Skip to content

Commit

Permalink
Add configs to enable running integration tests for cluster running b…
Browse files Browse the repository at this point in the history 
…ehind proxy (apache#3646)

* Add configs to enable running integration tests for cluster running behind proxy

As part of https://issues.apache.org/jira/browse/KNOX-758
I am working on adding support for proxying druid queries & UIs using
Apache KNOX gateway.
This PR adds support for integration-tests to be run using the proxy
gateway.
Changes Include -
1) Instead of hostName and port, ability to specify url in the config
file.
2) tests now use HTTPClient defined in DruidTestModule that can pass in
the request.

Note - the config changes are backwards compatible and existing configs
should work fine.

* review comments

* review comments
  • Loading branch information
@nishantmonu51 @b-slim
nishantmonu51 authored and b-slim committed Nov 4, 2016
1 parent b99e14e commit b961b6a
Show file tree
Hide file tree
Showing 14 changed files with 266 additions and 137 deletions.
117 changes: 78 additions & 39 deletions integration-tests/src/main/java/io/druid/testing/ConfigFileConfigProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;

import io.druid.java.util.common.logger.Logger;

import java.io.File;
Expand All @@ -33,18 +32,21 @@
public class ConfigFileConfigProvider implements IntegrationTestingConfigProvider
{
private final static Logger LOG = new Logger(ConfigFileConfigProvider.class);
private String routerHost = "";
private String brokerHost = "";
private String historicalHost = "";
private String coordinatorHost = "";
private String indexerHost = "";
private String middleManagerHost = "";
private String zookeeperHosts = ""; // comma-separated list of host:port
private String kafkaHost = "";
private String routerUrl;
private String brokerUrl;
private String historicalUrl;
private String coordinatorUrl;
private String indexerUrl;
private String middleManagerHost;
private String zookeeperHosts; // comma-separated list of host:port
private String kafkaHost;
private Map<String, String> props = null;
private String username;
private String password;

@JsonCreator
ConfigFileConfigProvider(@JsonProperty("configFile") String configFile){
ConfigFileConfigProvider(@JsonProperty("configFile") String configFile)
{
loadProperties(configFile);
}

Expand All @@ -53,69 +55,94 @@ private void loadProperties(String configFile)
ObjectMapper jsonMapper = new ObjectMapper();
try {
props = jsonMapper.readValue(
new File(configFile), new TypeReference<Map<String, String>>()
{
}
new File(configFile), new TypeReference<Map<String, String>>()
{
}
);
}
catch (IOException ex) {
throw new RuntimeException(ex);
}
// there might not be a router; we want routerHost to be null in that case
routerHost = props.get("router_host");
if (null != routerHost) {
routerHost += ":" + props.get("router_port");
routerUrl = props.get("router_url");
if (routerUrl == null) {
String routerHost = props.get("router_host");
if (null != routerHost) {
routerUrl = String.format("http://%s:%s", routerHost, props.get("router_port"));
}
}
brokerUrl = props.get("broker_url");
if (brokerUrl == null) {
brokerUrl = String.format("http://%s:%s", props.get("broker_host"), props.get("broker_port"));
}

historicalUrl = props.get("historical_url");
if (historicalUrl == null) {
historicalUrl = String.format("http://%s:%s", props.get("historical_host"), props.get("historical_port"));
}

coordinatorUrl = props.get("coordinator_url");
if (coordinatorUrl == null) {
coordinatorUrl = String.format("http://%s:%s", props.get("coordinator_host"), props.get("coordinator_port"));
}

indexerUrl = props.get("indexer_url");
if (indexerUrl == null) {
indexerUrl = String.format("http://%s:%s", props.get("indexer_host"), props.get("indexer_port"));
}
brokerHost = props.get("broker_host") + ":" + props.get("broker_port");
historicalHost = props.get("historical_host") + ":" + props.get("historical_port");
coordinatorHost = props.get("coordinator_host") + ":" + props.get("coordinator_port");
indexerHost = props.get("indexer_host") + ":" + props.get("indexer_port");
middleManagerHost = props.get("middlemanager_host");

zookeeperHosts = props.get("zookeeper_hosts");
kafkaHost = props.get("kafka_host") + ":" + props.get ("kafka_port");

LOG.info ("router: [%s]", routerHost);
LOG.info ("broker: [%s]", brokerHost);
LOG.info ("coordinator: [%s]", coordinatorHost);
LOG.info ("overlord: [%s]", indexerHost);
LOG.info ("middle manager: [%s]", middleManagerHost);
LOG.info ("zookeepers: [%s]", zookeeperHosts);
LOG.info ("kafka: [%s]", kafkaHost);
kafkaHost = props.get("kafka_host") + ":" + props.get("kafka_port");

username = props.get("username");

password = props.get("password");

LOG.info("router: [%s]", routerUrl);
LOG.info("broker: [%s]", brokerUrl);
LOG.info("coordinator: [%s]", coordinatorUrl);
LOG.info("overlord: [%s]", indexerUrl);
LOG.info("middle manager: [%s]", middleManagerHost);
LOG.info("zookeepers: [%s]", zookeeperHosts);
LOG.info("kafka: [%s]", kafkaHost);
LOG.info("Username: [%s]", username);
}

@Override
public IntegrationTestingConfig get()
{
return new IntegrationTestingConfig()
{

@Override
public String getCoordinatorHost()
public String getCoordinatorUrl()
{
return coordinatorHost;
return coordinatorUrl;
}

@Override
public String getIndexerHost()
public String getIndexerUrl()
{
return indexerHost;
return indexerUrl;
}

@Override
public String getRouterHost()
public String getRouterUrl()
{
return routerHost;
return routerUrl;
}

@Override
public String getBrokerHost()
public String getBrokerUrl()
{
return brokerHost;
return brokerUrl;
}

@Override
public String getHistoricalHost()
public String getHistoricalUrl()
{
return historicalHost;
return historicalUrl;
}

@Override
Expand All @@ -141,6 +168,18 @@ public String getProperty(String keyword)
{
return props.get(keyword);
}

@Override
public String getUsername()
{
return username;
}

@Override
public String getPassword()
{
return password;
}
};
}
}
37 changes: 25 additions & 12 deletions integration-tests/src/main/java/io/druid/testing/DockerConfigProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,33 +41,33 @@ public IntegrationTestingConfig get()
return new IntegrationTestingConfig()
{
@Override
public String getCoordinatorHost()
public String getCoordinatorUrl()
{
return dockerIp+":8081";
return "http://" + dockerIp + ":8081";
}

@Override
public String getIndexerHost()
public String getIndexerUrl()
{
return dockerIp+":8090";
return "http://" + dockerIp + ":8090";
}

@Override
public String getRouterHost()
public String getRouterUrl()
{
return dockerIp+ ":8888";
return "http://" + dockerIp + ":8888";
}

@Override
public String getBrokerHost()
public String getBrokerUrl()
{
return dockerIp + ":8082";
return "http://" + dockerIp + ":8082";
}

@Override
public String getHistoricalHost()
public String getHistoricalUrl()
{
return dockerIp + ":8083";
return "http://" + dockerIp + ":8083";
}

@Override
Expand All @@ -88,14 +88,27 @@ public String getKafkaHost()
return dockerIp + ":9092";
}


@Override
public String getProperty(String prop)
{
if (prop.equals("hadoopTestDir")) {
return hadoopDir;
}
return hadoopDir;
}
throw new UnsupportedOperationException("DockerConfigProvider does not support property " + prop);
}

@Override
public String getUsername()
{
return null;
}

@Override
public String getPassword()
{
return null;
}
};
}
}
14 changes: 9 additions & 5 deletions integration-tests/src/main/java/io/druid/testing/IntegrationTestingConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,15 @@
*/
public interface IntegrationTestingConfig
{
public String getCoordinatorHost();
public String getCoordinatorUrl();

public String getIndexerHost();
public String getIndexerUrl();

public String getRouterHost();
public String getRouterUrl();

public String getBrokerHost();
public String getBrokerUrl();

public String getHistoricalHost();
public String getHistoricalUrl();

public String getMiddleManagerHost();

Expand All @@ -40,4 +40,8 @@ public interface IntegrationTestingConfig
public String getKafkaHost();

public String getProperty(String prop);

String getUsername();

String getPassword();
}
12 changes: 6 additions & 6 deletions integration-tests/src/main/java/io/druid/testing/clients/ClientInfoResourceTestClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@
import com.metamx.http.client.Request;
import com.metamx.http.client.response.StatusResponseHandler;
import com.metamx.http.client.response.StatusResponseHolder;
import io.druid.guice.annotations.Global;
import io.druid.java.util.common.ISE;
import io.druid.testing.IntegrationTestingConfig;
import io.druid.testing.guice.TestClient;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpResponseStatus;

Expand All @@ -41,27 +41,27 @@ public class ClientInfoResourceTestClient
{
private final ObjectMapper jsonMapper;
private final HttpClient httpClient;
private final String broker;
private final String brokerUrl;
private final StatusResponseHandler responseHandler;

@Inject
ClientInfoResourceTestClient(
ObjectMapper jsonMapper,
@Global HttpClient httpClient,
@TestClient HttpClient httpClient,
IntegrationTestingConfig config
)
{
this.jsonMapper = jsonMapper;
this.httpClient = httpClient;
this.broker = config.getBrokerHost();
this.brokerUrl = config.getBrokerUrl();
this.responseHandler = new StatusResponseHandler(Charsets.UTF_8);
}

private String getBrokerURL()
{
return String.format(
"http://%s/druid/v2/datasources",
broker
"%s/druid/v2/datasources",
brokerUrl
);
}

Expand Down
10 changes: 5 additions & 5 deletions integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@
import com.metamx.http.client.Request;
import com.metamx.http.client.response.StatusResponseHandler;
import com.metamx.http.client.response.StatusResponseHolder;
import io.druid.guice.annotations.Global;
import io.druid.java.util.common.ISE;
import io.druid.testing.IntegrationTestingConfig;
import io.druid.testing.guice.TestClient;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpResponseStatus;
import org.joda.time.Interval;
Expand All @@ -48,20 +48,20 @@ public class CoordinatorResourceTestClient

@Inject
CoordinatorResourceTestClient(
ObjectMapper jsonMapper,
@Global HttpClient httpClient, IntegrationTestingConfig config
ObjectMapper jsonMapper,
@TestClient HttpClient httpClient, IntegrationTestingConfig config
)
{
this.jsonMapper = jsonMapper;
this.httpClient = httpClient;
this.coordinator = config.getCoordinatorHost();
this.coordinator = config.getCoordinatorUrl();
this.responseHandler = new StatusResponseHandler(Charsets.UTF_8);
}

private String getCoordinatorURL()
{
return String.format(
"http://%s/druid/coordinator/v1/",
"%s/druid/coordinator/v1/",
coordinator
);
}
Expand Down
5 changes: 2 additions & 3 deletions ...gration-tests/src/main/java/io/druid/testing/clients/EventReceiverFirehoseTestClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,8 @@
import com.metamx.http.client.Request;
import com.metamx.http.client.response.StatusResponseHandler;
import com.metamx.http.client.response.StatusResponseHolder;

import io.druid.java.util.common.ISE;

import io.druid.testing.guice.TestClient;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpResponseStatus;

Expand All @@ -55,7 +54,7 @@ public EventReceiverFirehoseTestClient(
String host,
String chatID,
ObjectMapper jsonMapper,
HttpClient httpClient,
@TestClient HttpClient httpClient,
ObjectMapper smileMapper
)
{
Expand Down
Loading

0 comments on commit b961b6a

Please sign in to comment.