Skip to content

Commit

Permalink
KYLIN-3496 Make calcite extras props available in JDBC Driver
Browse files Browse the repository at this point in the history
  • Loading branch information
@ian4hu @shaofengshi
ian4hu authored and shaofengshi committed Oct 12, 2018
1 parent e2c1e7e commit 8581322
Show file tree
Hide file tree
Showing 6 changed files with 183 additions and 42 deletions.
39 changes: 39 additions & 0 deletions core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,13 @@

package org.apache.kylin.common.debug;

import java.io.IOException;
import java.io.StringReader;
import java.util.Map;
import java.util.Properties;
import java.util.Set;

import com.google.common.collect.Sets;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;

Expand Down Expand Up @@ -156,6 +161,35 @@ public static void cleanToggles() {
_backdoorToggles.remove();
}

/**
* get extra calcite props from jdbc client
*/
public static Properties getJdbcDriverClientCalciteProps() {
Properties props = new Properties();
String propsStr = getString(JDBC_CLIENT_CALCITE_PROPS);
if (propsStr == null) {
return props;
}
try {
props.load(new StringReader(propsStr));
} catch (IOException ignored) {
// ignored
}
final Set<String> allowedPropsNames = Sets.newHashSet(
"caseSensitive",
"unquotedCasing",
"quoting",
"conformance"
);
// remove un-allowed props
for (String key : props.stringPropertyNames()) {
if (!allowedPropsNames.contains(key)) {
props.remove(key);
}
}
return props;
}

/**
* set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage
*
Expand Down Expand Up @@ -313,4 +347,9 @@ public static void cleanToggles() {
}
*/
public final static String DEBUG_TOGGLE_HTRACE_ENABLED = "DEBUG_TOGGLE_HTRACE_ENABLED";

/**
* extra calcite props from jdbc client
*/
public static final String JDBC_CLIENT_CALCITE_PROPS = "JDBC_CLIENT_CALCITE_PROPS";
}
25 changes: 20 additions & 5 deletions jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@

import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Set;

import com.google.common.collect.Sets;
import org.apache.calcite.avatica.AvaticaConnection;
import org.apache.calcite.avatica.DriverVersion;
import org.apache.calcite.avatica.Meta;
Expand All @@ -40,8 +42,8 @@
* Supported Statements:
* </p>
* <ul>
* <li>{@link KylinStatementImpl}</li>
* <li>{@link KylinPrepareStatementImpl}</li>
* <li>{@link KylinStatement}</li>
* <li>{@link KylinPreparedStatement}</li>
* </ul>
*
* <p>
Expand All @@ -50,6 +52,7 @@
* <li>user: username</li>
* <li>password: password</li>
* <li>ssl: true/false</li>
* <li>{@link #CLIENT_CALCITE_PROP_NAMES extras calcite props} like: caseSensitive, unquotedCasing, quoting, conformance</li>
* </ul>
* </p>
*
Expand All @@ -59,9 +62,10 @@
* <pre>
* Driver driver = (Driver) Class.forName(&quot;org.apache.kylin.kylin.jdbc.Driver&quot;).newInstance();
* Properties info = new Properties();
* info.put(&quot;user&quot;, &quot;user&quot;);
* info.put(&quot;password&quot;, &quot;password&quot;);
* info.put(&quot;ssl&quot;, true);
* info.setProperty(&quot;user&quot;, &quot;user&quot;);
* info.setProperty(&quot;password&quot;, &quot;password&quot;);
* info.setProperty(&quot;ssl&quot;, &quot;true&quot;);
* info.setProperty(&quot;caseSensitive&quot;, &quot;true&quot;);
* Connection conn = driver.connect(&quot;jdbc:kylin://{domain}/{project}&quot;, info);
* </pre>
*
Expand All @@ -70,6 +74,17 @@
public class Driver extends UnregisteredDriver {

public static final String CONNECT_STRING_PREFIX = "jdbc:kylin:";

/**
* These calcite props can be configured by jdbc connection
*/
public static final Set<String> CLIENT_CALCITE_PROP_NAMES = Sets.newHashSet(
"caseSensitive",
"unquotedCasing",
"quoting",
"conformance"
);

static {
try {
DriverManager.registerDriver(new Driver());
Expand Down
2 changes: 1 addition & 1 deletion jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@ private SQLResponseStub executeKylinQuery(String sql, List<StatementParameter> p
addHttpHeaders(post);

String postBody = jsonMapper.writeValueAsString(request);
logger.debug("Post body:\n " + postBody);
logger.debug("Post body:\n {}", postBody);
StringEntity requestEntity = new StringEntity(postBody, ContentType.create("application/json", "UTF-8"));
post.setEntity(requestEntity);

Expand Down
32 changes: 31 additions & 1 deletion jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@
package org.apache.kylin.jdbc;

import java.io.IOException;
import java.io.StringWriter;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.TimeZone;

import org.apache.calcite.avatica.AvaticaParameter;
Expand Down Expand Up @@ -58,11 +60,13 @@ protected AvaticaResultSet execute() throws SQLException {
paramValues = ((KylinPreparedStatement) statement).getParameterJDBCValues();
}

IRemoteClient client = ((KylinConnection) statement.connection).getRemoteClient();
KylinConnection connection = (KylinConnection) statement.connection;
IRemoteClient client = connection.getRemoteClient();

Map<String, String> queryToggles = new HashMap<>();
int maxRows = statement.getMaxRows();
queryToggles.put("ATTR_STATEMENT_MAX_ROWS", String.valueOf(maxRows));
addServerProps(queryToggles, connection);

QueryResult result;
try {
Expand All @@ -78,4 +82,30 @@ protected AvaticaResultSet execute() throws SQLException {
return super.execute2(cursor, columnMetaDataList);
}

/**
* add calcite props into queryToggles
*/
private void addServerProps(Map<String, String> queryToggles, KylinConnection connection) {
Properties connProps = connection.getConnectionProperties();
Properties props = new Properties();
for (String key : connProps.stringPropertyNames()) {
if (Driver.CLIENT_CALCITE_PROP_NAMES.contains(key)) {
props.put(key, connProps.getProperty(key));
}
}

if (props.isEmpty()) {
return;
}

StringWriter writer = new StringWriter();
try {
props.store(writer, "");
} catch (IOException ignored) {
// ignored
return;
}
queryToggles.put("JDBC_CLIENT_CALCITE_PROPS", writer.toString());
}

}
124 changes: 89 additions & 35 deletions jdbc/src/test/java/org/apache/kylin/jdbc/KylinConnectionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,14 @@
package org.apache.kylin.jdbc;

import java.io.IOException;
import java.io.StringReader;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Types;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
Expand All @@ -36,13 +38,18 @@
import org.apache.http.message.BasicStatusLine;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentMatcher;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;

import javax.annotation.Nonnull;

import static org.apache.http.HttpVersion.HTTP_1_1;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
import static org.mockito.Mockito.doAnswer;
Expand All @@ -55,10 +62,10 @@

public class KylinConnectionTest {

private Driver driver = new Driver();
private KylinJdbcFactory factory = spy(new KylinJdbcFactory.Version41());
private IRemoteClient client = mock(IRemoteClient.class);
private HttpClient httpClient = mock(HttpClient.class);
private final Driver driver = new Driver();
private final KylinJdbcFactory factory = spy(new KylinJdbcFactory.Version41());
private final IRemoteClient client = mock(IRemoteClient.class);
private final HttpClient httpClient = mock(HttpClient.class);

@Before
public void setUp() throws Exception {
Expand All @@ -68,54 +75,50 @@ public void setUp() throws Exception {
@Test
public void testPrepareStatementWithMockKylinClient() throws SQLException, IOException {
String sql = "select 1 as val";
ArrayList<ColumnMetaData> columnMeta = new ArrayList<>();
columnMeta.add(new ColumnMetaData(0, false, true, false,
false, 1, true, 1,
"VAL", "VAL", null,
10, 0, null, null,
ColumnMetaData.scalar(Types.INTEGER, "INTEGER", ColumnMetaData.Rep.INTEGER),
true, false, false, "java.lang.Integer"));
ArrayList<Object> list = new ArrayList<>();
list.add(new Object[]{1});
IRemoteClient.QueryResult result = new IRemoteClient.QueryResult(columnMeta, list);
// mock client
when(client.executeQuery(anyString(), Mockito.<List<Object>>any(), Mockito.<Map<String, String>>any())).thenReturn(result);
when(client.executeQuery(anyString(), Mockito.<List<Object>>any(), Mockito.<Map<String, String>>any())).thenReturn(getMockResult());

PreparedStatement preparedStatement = getConnectionWithMockClient().prepareStatement(sql);
ResultSet resultSet = preparedStatement.executeQuery();
try (KylinConnection conn = getConnectionWithMockClient()) {
PreparedStatement preparedStatement = conn.prepareStatement(sql);
try (ResultSet resultSet = preparedStatement.executeQuery()) {
verify(client).executeQuery(eq(sql), Mockito.<List<Object>>any(), Mockito.<Map<String, String>>any());

verify(client).executeQuery(eq(sql), Mockito.<List<Object>>any(), Mockito.<Map<String, String>>any());

assertTrue(resultSet.next());
ResultSetMetaData metaData = resultSet.getMetaData();
assertEquals("VAL", metaData.getColumnName(1));
assertEquals(1, resultSet.getInt("VAL"));
assertTrue(resultSet.next());
ResultSetMetaData metaData = resultSet.getMetaData();
assertEquals("VAL", metaData.getColumnName(1));
assertEquals(1, resultSet.getInt("VAL"));
}
}
}

@Test
public void testPrepareStatementWithMockHttp() throws IOException, SQLException {
String sql = "select 1 as val";
KylinConnection connection = getConnectionWithMockHttp();
try (KylinConnection connection = getConnectionWithMockHttp()) {

// mock http
HttpResponse response = TestUtil.mockHttpResponseWithFile(200, "OK", "query.json");
when(httpClient.execute(any(HttpUriRequest.class))).thenReturn(response);
// mock http
HttpResponse response = TestUtil.mockHttpResponseWithFile(200, "OK", "query.json");
when(httpClient.execute(any(HttpUriRequest.class))).thenReturn(response);

ResultSet resultSet = connection.prepareStatement(sql).executeQuery();

assertTrue(resultSet.next());
ResultSetMetaData metaData = resultSet.getMetaData();
assertEquals("VAL", metaData.getColumnName(1));
assertEquals(1, resultSet.getInt("VAL"));
try (ResultSet resultSet = connection.prepareStatement(sql).executeQuery()) {
assertTrue(resultSet.next());
ResultSetMetaData metaData = resultSet.getMetaData();
assertEquals("VAL", metaData.getColumnName(1));
assertEquals(1, resultSet.getInt("VAL"));
}
}
}

private KylinConnection getConnectionWithMockClient() throws SQLException {
Properties info = new Properties();
return getConnectionWithMockClient("jdbc:kylin:test_url/test_db", new Properties());
}

private KylinConnection getConnectionWithMockClient(String url, @Nonnull Properties info) throws SQLException {
info.setProperty("user", "ADMIN");
info.setProperty("password", "KYLIN");

doReturn(client).when(factory).newRemoteClient(any(KylinConnectionInfo.class));
return new KylinConnection(driver, factory, "jdbc:kylin:test_url/test_db", info);
return new KylinConnection(driver, factory, url, info);
}

private KylinConnection getConnectionWithMockHttp() throws SQLException, IOException {
Expand All @@ -141,4 +144,55 @@ public Object answer(InvocationOnMock invo) throws Throwable {

return new KylinConnection(driver, factory, "jdbc:kylin:test_url/test_db", info);
}

@Test
public void testJdbcClientCalcitePropsInUrl() throws Exception {
String sql = "select 1 as val";

// mock client
when(client.executeQuery(anyString(), Mockito.<List<Object>>any(), Mockito.<Map<String, String>>any())).thenReturn(getMockResult());
Map<String, String> toggles = new HashMap<>();
Properties info = new Properties();
info.setProperty("caseSensitive", "false");
info.setProperty("unquotedCasing", "UNCHANGED");
try (KylinConnection conn = getConnectionWithMockClient("jdbc:kylin:test_url/test_db", info)) {
PreparedStatement preparedStatement = conn.prepareStatement(sql);
try (ResultSet resultSet = preparedStatement.executeQuery()) {
verify(client).executeQuery(eq(sql), Mockito.<List<Object>>any(), argThat(new ArgumentMatcher<Map<String, String>>() {
@Override
public boolean matches(Map<String, String> argument) {
String propsStr = argument.get("JDBC_CLIENT_CALCITE_PROPS");
assertNotNull(propsStr);
Properties props = new Properties();
try {
props.load(new StringReader(propsStr));
} catch (IOException e) {
throw new RuntimeException(e);
}
assertEquals("false", props.getProperty("caseSensitive"));
assertEquals("UNCHANGED", props.getProperty("unquotedCasing"));
return true;
}
}));

assertTrue(resultSet.next());
ResultSetMetaData metaData = resultSet.getMetaData();
assertEquals("VAL", metaData.getColumnName(1));
assertEquals(1, resultSet.getInt("VAL"));
}
}
}

private IRemoteClient.QueryResult getMockResult() {
ArrayList<ColumnMetaData> columnMeta = new ArrayList<>();
columnMeta.add(new ColumnMetaData(0, false, true, false,
false, 1, true, 1,
"VAL", "VAL", null,
10, 0, null, null,
ColumnMetaData.scalar(Types.INTEGER, "INTEGER", ColumnMetaData.Rep.INTEGER),
true, false, false, "java.lang.Integer"));
ArrayList<Object> list = new ArrayList<>();
list.add(new Object[]{1});
return new IRemoteClient.QueryResult(columnMeta, list);
}
}
3 changes: 3 additions & 0 deletions query/src/main/java/org/apache/kylin/query/QueryConnection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

import org.apache.calcite.jdbc.Driver;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.query.schema.OLAPSchemaFactory;

public class QueryConnection {
Expand All @@ -47,6 +48,8 @@ public static Connection getConnection(String project) throws SQLException {
File olapTmp = OLAPSchemaFactory.createTempOLAPJson(project, KylinConfig.getInstanceFromEnv());
Properties info = new Properties();
info.putAll(KylinConfig.getInstanceFromEnv().getCalciteExtrasProperties());
// Import calcite props from jdbc client(override the kylin.properties)
info.putAll(BackdoorToggles.getJdbcDriverClientCalciteProps());
info.put("model", olapTmp.getAbsolutePath());
info.put("typeSystem", "org.apache.kylin.query.calcite.KylinRelDataTypeSystem");
return DriverManager.getConnection("jdbc:calcite:", info);
Expand Down

0 comments on commit 8581322

Please sign in to comment.