Skip to content

Commit

Permalink
sweep test environment (apache#10017)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dmsolr
dmsolr authored Apr 10, 2021
1 parent ac1de00 commit 3a6225d
Show file tree
Hide file tree
Showing 25 changed files with 93 additions and 885 deletions.
133 changes: 5 additions & 128 deletions .../test/java/org/apache/shardingsphere/test/integration/env/IntegrationTestEnvironment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,10 @@
import lombok.Getter;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.infra.database.type.DatabaseTypeRegistry;
import org.apache.shardingsphere.infra.database.type.dialect.H2DatabaseType;
import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType;
import org.apache.shardingsphere.test.integration.env.database.embedded.EmbeddedDatabaseDistributionProperties;
import org.apache.shardingsphere.test.integration.env.database.embedded.EmbeddedDatabaseManager;
import org.apache.shardingsphere.test.integration.env.datasource.DataSourceEnvironment;
import org.apache.shardingsphere.test.integration.env.props.DatabaseScenarioProperties;
import org.apache.shardingsphere.test.integration.env.props.EnvironmentProperties;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
Expand All @@ -59,24 +45,15 @@ public final class IntegrationTestEnvironment {

private final boolean runAdditionalTestCases;

private Map<DatabaseType, Map<String, DataSourceEnvironment>> dataSourceEnvironments;

private final Map<String, DataSourceEnvironment> proxyEnvironments;
private final Set<DatabaseType> databaseTypes;

private IntegrationTestEnvironment() {
Properties engineEnvProps = EnvironmentProperties.loadProperties("env/engine-env.properties");
envType = getEnvironmentType(engineEnvProps);
adapters = Splitter.on(",").trimResults().splitToList(engineEnvProps.getProperty("it.adapters"));
scenarios = getScenarios(engineEnvProps);
runAdditionalTestCases = Boolean.parseBoolean(engineEnvProps.getProperty("it.run.additional.cases"));
Map<String, DatabaseScenarioProperties> databaseProps = getDatabaseScenarioProperties();
dataSourceEnvironments = createDataSourceEnvironments(getDatabaseTypes(engineEnvProps), databaseProps);
if (EnvironmentType.EMBEDDED == envType) {
EmbeddedDatabaseDistributionProperties embeddedDatabaseProps = new EmbeddedDatabaseDistributionProperties(EnvironmentProperties.loadProperties("env/embedded-databases.properties"));
dataSourceEnvironments = mergeDataSourceEnvironments(embeddedDatabaseProps);
createEmbeddedDatabases(embeddedDatabaseProps);
}
proxyEnvironments = createProxyEnvironments(databaseProps);
databaseTypes = getDatabaseTypes(engineEnvProps);
}

private EnvironmentType getEnvironmentType(final Properties engineEnvProps) {
Expand All @@ -95,109 +72,8 @@ private Collection<String> getScenarios(final Properties engineEnvProps) {
return result;
}

private Map<String, DatabaseScenarioProperties> getDatabaseScenarioProperties() {
Map<String, DatabaseScenarioProperties> result = new HashMap<>(scenarios.size(), 1);
for (String each : scenarios) {
result.put(each, new DatabaseScenarioProperties(each, EnvironmentProperties.loadProperties(String.format("env/%s/scenario-env.properties", each))));
}
return result;
}

private Collection<DatabaseType> getDatabaseTypes(final Properties engineEnvProps) {
return Arrays.stream(engineEnvProps.getProperty("it.databases").split(",")).map(each -> DatabaseTypeRegistry.getActualDatabaseType(each.trim())).collect(Collectors.toList());
}

private Map<DatabaseType, Map<String, DataSourceEnvironment>> createDataSourceEnvironments(final Collection<DatabaseType> databaseTypes,
final Map<String, DatabaseScenarioProperties> databaseProps) {
Map<DatabaseType, Map<String, DataSourceEnvironment>> result = new LinkedHashMap<>(databaseTypes.size(), 1);
for (DatabaseType each : databaseTypes) {
Map<String, DataSourceEnvironment> dataSourceEnvs = new LinkedHashMap<>(scenarios.size(), 1);
for (String scenario : scenarios) {
dataSourceEnvs.put(scenario, createDataSourceEnvironment(each, databaseProps.get(scenario)));
result.put(each, dataSourceEnvs);
}
}
return result;
}

private DataSourceEnvironment createDataSourceEnvironment(final DatabaseType databaseType, final DatabaseScenarioProperties databaseProps) {
if (databaseType instanceof H2DatabaseType) {
return new DataSourceEnvironment(databaseType, "", 0, "sa", "");
}
return new DataSourceEnvironment(databaseType, databaseProps.getDatabaseHost(databaseType),
databaseProps.getDatabasePort(databaseType), databaseProps.getDatabaseUsername(databaseType), databaseProps.getDatabasePassword(databaseType));
}

private Map<DatabaseType, Map<String, DataSourceEnvironment>> mergeDataSourceEnvironments(final EmbeddedDatabaseDistributionProperties embeddedDatabaseProps) {
Set<DatabaseType> databaseTypes = dataSourceEnvironments.keySet();
Map<DatabaseType, Map<String, DataSourceEnvironment>> result = new LinkedHashMap<>(databaseTypes.size(), 1);
for (DatabaseType each : databaseTypes) {
Map<String, DataSourceEnvironment> dataSourceEnvs = new LinkedHashMap<>(scenarios.size(), 1);
for (String scenario : scenarios) {
dataSourceEnvs.put(scenario, mergeDataSourceEnvironment(dataSourceEnvironments.get(each).get(scenario), embeddedDatabaseProps));
result.put(each, dataSourceEnvs);
}
}
return result;
}

private DataSourceEnvironment mergeDataSourceEnvironment(final DataSourceEnvironment dataSourceEnvironment, final EmbeddedDatabaseDistributionProperties embeddedDatabaseProps) {
DatabaseType databaseType = dataSourceEnvironment.getDatabaseType();
if (databaseType instanceof H2DatabaseType) {
return new DataSourceEnvironment(databaseType, "", 0, "sa", "");
}
return new DataSourceEnvironment(databaseType, dataSourceEnvironment.getHost(),
embeddedDatabaseProps.getInstancePort(databaseType), dataSourceEnvironment.getUsername(), dataSourceEnvironment.getPassword());
}

private void createEmbeddedDatabases(final EmbeddedDatabaseDistributionProperties embeddedDatabaseProps) {
for (Entry<DatabaseType, Map<String, DataSourceEnvironment>> entry : dataSourceEnvironments.entrySet()) {
createEmbeddedDatabases(entry.getKey(), entry.getValue(), embeddedDatabaseProps);
}
}

private void createEmbeddedDatabases(final DatabaseType databaseType,
final Map<String, DataSourceEnvironment> dataSourceEnvs, final EmbeddedDatabaseDistributionProperties embeddedDatabaseProps) {
for (Entry<String, DataSourceEnvironment> entry : dataSourceEnvs.entrySet()) {
EmbeddedDatabaseManager.startUp(databaseType.getName(), entry.getKey(), embeddedDatabaseProps, entry.getValue().getPort());
}
}

private Map<String, DataSourceEnvironment> createProxyEnvironments(final Map<String, DatabaseScenarioProperties> databaseProps) {
Map<String, DataSourceEnvironment> result = new HashMap<>(scenarios.size(), 1);
for (String each : scenarios) {
result.put(each, createProxyEnvironment(databaseProps.get(each)));
}
return result;
}

private DataSourceEnvironment createProxyEnvironment(final DatabaseScenarioProperties databaseProps) {
// TODO hard code for MySQL, should configurable
return new DataSourceEnvironment(
new MySQLDatabaseType(), databaseProps.getProxyHost(), databaseProps.getProxyPort(), databaseProps.getProxyUsername(), databaseProps.getProxyPassword());
}

private void waitForEnvironmentReady(final String scenario) {
int retryCount = 0;
while (!isProxyReady(scenario) && retryCount < 30) {
try {
Thread.sleep(1000L);
} catch (final InterruptedException ignore) {
}
retryCount++;
}
}

@SuppressWarnings("CallToDriverManagerGetConnection")
private boolean isProxyReady(final String scenario) {
DataSourceEnvironment dataSourceEnv = proxyEnvironments.get(scenario);
try (Connection connection = DriverManager.getConnection(dataSourceEnv.getURL(scenario), dataSourceEnv.getUsername(), dataSourceEnv.getPassword());
Statement statement = connection.createStatement()) {
statement.execute("SELECT 1");
} catch (final SQLException ignore) {
return false;
}
return true;
private Set<DatabaseType> getDatabaseTypes(final Properties engineEnvProps) {
return Arrays.stream(engineEnvProps.getProperty("it.databases").split(",")).map(each -> DatabaseTypeRegistry.getActualDatabaseType(each.trim())).collect(Collectors.toSet());
}

/**
Expand All @@ -208,4 +84,5 @@ private boolean isProxyReady(final String scenario) {
public static IntegrationTestEnvironment getInstance() {
return INSTANCE;
}

}
47 changes: 1 addition & 46 deletions ...a/org/apache/shardingsphere/test/integration/env/database/DatabaseEnvironmentManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,15 @@

import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader;
import org.apache.shardingsphere.infra.spi.typed.TypedSPIRegistry;
import org.apache.shardingsphere.test.integration.env.EnvironmentPath;
import org.apache.shardingsphere.test.integration.env.IntegrationTestEnvironment;
import org.apache.shardingsphere.test.integration.env.database.initialization.DatabaseSQLInitialization;
import org.h2.tools.RunScript;

import javax.sql.DataSource;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Collection;
import java.util.Properties;

/**
* Database environment manager.
Expand All @@ -50,7 +41,7 @@ public final class DatabaseEnvironmentManager {

/**
* Get database names.
*
*
* @param scenario scenario
* @return database names
* @throws IOException IO exception
Expand All @@ -66,40 +57,4 @@ private static DatabaseNameEnvironment unmarshal(final String databasesFile) thr
}
}

/**
* Execute init SQLs.
*
* @throws IOException IO exception
* @throws JAXBException JAXB exception
* @throws SQLException SQL exception
*/
public static void executeInitSQLs() throws IOException, JAXBException, SQLException {
for (String each : IntegrationTestEnvironment.getInstance().getScenarios()) {
executeInitSQLs(each);
}
}

private static void executeInitSQLs(final String scenario) throws IOException, JAXBException, SQLException {
for (DatabaseType each : IntegrationTestEnvironment.getInstance().getDataSourceEnvironments().keySet()) {
DatabaseSQLInitialization databaseSQLInitialization = TypedSPIRegistry.getRegisteredService(DatabaseSQLInitialization.class, each.getName(), new Properties());
databaseSQLInitialization.executeInitSQLs(scenario, each);
}
}

/**
* Execute SQL script.
*
* @param dataSource data source
* @param file script file
*
* @throws SQLException SQL exception
* @throws IOException IO exception
*/
public static void executeSQLScript(final DataSource dataSource, final File file) throws SQLException, IOException {
try (Connection connection = dataSource.getConnection();
FileReader reader = new FileReader(file)) {
// TODO If you don't use H2 in the future, you need to implement this method.
RunScript.execute(connection, reader);
}
}
}
5 changes: 4 additions & 1 deletion ...hardingsphere/test/integration/env/database/initialization/DatabaseSQLInitialization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.infra.spi.typed.TypedSPI;

import javax.sql.DataSource;
import javax.xml.bind.JAXBException;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Map;

/**
* Database SQL initialization.
Expand All @@ -34,10 +36,11 @@ public interface DatabaseSQLInitialization extends TypedSPI {
*
* @param scenario scenario
* @param databaseType database type
* @param dataSourceMap datasource map
*
* @throws IOException IO exception
* @throws SQLException SQL exception
* @throws JAXBException JAXB exception
*/
void executeInitSQLs(String scenario, DatabaseType databaseType) throws IOException, SQLException, JAXBException;
void executeInitSQLs(String scenario, DatabaseType databaseType, Map<String, DataSource> dataSourceMap) throws IOException, SQLException, JAXBException;
}
19 changes: 13 additions & 6 deletions ...e/test/integration/env/database/initialization/type/DefaultDatabaseSQLInitialization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,15 @@

import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.test.integration.env.EnvironmentPath;
import org.apache.shardingsphere.test.integration.env.database.DatabaseEnvironmentManager;
import org.apache.shardingsphere.test.integration.env.datasource.builder.ActualDataSourceBuilder;
import org.h2.tools.RunScript;

import javax.sql.DataSource;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Map;

/**
* Database SQL initialization for Default.
Expand All @@ -37,14 +39,19 @@ public abstract class DefaultDatabaseSQLInitialization {
*
* @param scenario scenario
* @param databaseType database type
* @param dataSourceMap datasource map
*
* @throws IOException IO exception
* @throws SQLException SQL exception
*/
public void executeInitSQLs(final String scenario, final DatabaseType databaseType) throws IOException, SQLException {
// TODO use multiple threads to improve performance
DataSource dataSource = ActualDataSourceBuilder.build(null, scenario, databaseType);
public void executeInitSQLs(final String scenario, final DatabaseType databaseType, final Map<String, DataSource> dataSourceMap) throws IOException, SQLException {
File file = new File(EnvironmentPath.getInitSQLFile(databaseType, scenario));
DatabaseEnvironmentManager.executeSQLScript(dataSource, file);
// TODO use multiple threads to improve performance
for (Map.Entry<String, DataSource> each : dataSourceMap.entrySet()) {
try (Connection connection = each.getValue().getConnection();
FileReader reader = new FileReader(file)) {
RunScript.execute(connection, reader);
}
}
}
}
18 changes: 11 additions & 7 deletions ...sphere/test/integration/env/database/initialization/type/H2DatabaseSQLInitialization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,28 +20,32 @@
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.infra.database.type.dialect.H2DatabaseType;
import org.apache.shardingsphere.test.integration.env.EnvironmentPath;
import org.apache.shardingsphere.test.integration.env.database.DatabaseEnvironmentManager;
import org.apache.shardingsphere.test.integration.env.database.initialization.DatabaseSQLInitialization;
import org.apache.shardingsphere.test.integration.env.datasource.builder.ActualDataSourceBuilder;
import org.h2.tools.RunScript;

import javax.sql.DataSource;
import javax.xml.bind.JAXBException;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Map;

/**
* Database SQL initialization for H2.
*/
public final class H2DatabaseSQLInitialization implements DatabaseSQLInitialization {

@Override
public void executeInitSQLs(final String scenario, final DatabaseType databaseType) throws IOException, JAXBException, SQLException {
public void executeInitSQLs(final String scenario, final DatabaseType databaseType, final Map<String, DataSource> dataSourceMap) throws IOException, JAXBException, SQLException {
File file = new File(EnvironmentPath.getInitSQLFile(databaseType, scenario));
for (String each : DatabaseEnvironmentManager.getDatabaseNames(scenario)) {
// TODO use multiple threads to improve performance
DataSource dataSource = ActualDataSourceBuilder.build(each, scenario, databaseType);
DatabaseEnvironmentManager.executeSQLScript(dataSource, file);
// TODO use multiple threads to improve performance
for (Map.Entry<String, DataSource> each : dataSourceMap.entrySet()) {
try (Connection connection = each.getValue().getConnection();
FileReader reader = new FileReader(file)) {
RunScript.execute(connection, reader);
}
}
}

Expand Down
6 changes: 4 additions & 2 deletions ...e/test/integration/env/database/initialization/type/MariaDBDatabaseSQLInitialization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,19 @@
import org.apache.shardingsphere.infra.database.type.dialect.MariaDBDatabaseType;
import org.apache.shardingsphere.test.integration.env.database.initialization.DatabaseSQLInitialization;

import javax.sql.DataSource;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Map;

/**
* Database SQL initialization for MariaDB.
*/
public final class MariaDBDatabaseSQLInitialization extends DefaultDatabaseSQLInitialization implements DatabaseSQLInitialization {

@Override
public void executeInitSQLs(final String scenario, final DatabaseType databaseType) throws IOException, SQLException {
super.executeInitSQLs(scenario, databaseType);
public void executeInitSQLs(final String scenario, final DatabaseType databaseType, final Map<String, DataSource> dataSourceMap) throws IOException, SQLException {
super.executeInitSQLs(scenario, databaseType, dataSourceMap);
}

@Override
Expand Down
6 changes: 4 additions & 2 deletions ...ere/test/integration/env/database/initialization/type/MySQLDatabaseSQLInitialization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,19 @@
import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType;
import org.apache.shardingsphere.test.integration.env.database.initialization.DatabaseSQLInitialization;

import javax.sql.DataSource;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Map;

/**
* Database SQL initialization for MySQL.
*/
public final class MySQLDatabaseSQLInitialization extends DefaultDatabaseSQLInitialization implements DatabaseSQLInitialization {

@Override
public void executeInitSQLs(final String scenario, final DatabaseType databaseType) throws IOException, SQLException {
super.executeInitSQLs(scenario, databaseType);
public void executeInitSQLs(final String scenario, final DatabaseType databaseType, final Map<String, DataSource> dataSourceMap) throws IOException, SQLException {
super.executeInitSQLs(scenario, databaseType, dataSourceMap);
}

@Override
Expand Down
Loading

0 comments on commit 3a6225d

Please sign in to comment.