Add SSL option for Postgres source/destination (airbytehq#2757)

* add ssl for source-postgres * add config in utf8 test * correct comments from @jrhizor and @sherifnada * correct config get * add ssl test postgres * add sh generate ssl files * change pg ssl test * use custom image * correct spec.json * correc tests * remove unecessary config * add config and correct spec.json * add ssl to postgres destination * add tools to generate custom dockers images and correct spec.json * change how additional parameter is append * add logic ssl for postgres destination * remove if for append add params * gradlew format
Jimao · Apr 16, 2021 · ca8f304 · ca8f304
1 parent e4326aa
commit ca8f304
Show file tree

Hide file tree

Showing 15 changed files with 310 additions and 11 deletions.
diff --git a/...tgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/...tgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java
@@ -31,6 +31,8 @@
 import io.airbyte.integrations.base.IntegrationRunner;
 import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination;
 import io.airbyte.integrations.destination.jdbc.DefaultSqlOperations;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Optional;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,12 +51,25 @@ public PostgresDestination() {
   public JsonNode toJdbcConfig(JsonNode config) {
     final String schema = Optional.ofNullable(config.get("schema")).map(JsonNode::asText).orElse("public");
 
+    List<String> additionalParameters = new ArrayList<>();
+
+    final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:postgresql://%s:%s/%s?",
+        config.get("host").asText(),
+        config.get("port").asText(),
+        config.get("database").asText()));
+
+    if (config.get("ssl").asBoolean()) {
+      additionalParameters.add("ssl=true");
+      additionalParameters.add("sslmode=require");
+    }
+
+    if (!additionalParameters.isEmpty()) {
+      additionalParameters.forEach(x -> jdbcUrl.append(x).append("&"));
+    }
+
     final ImmutableMap.Builder<Object, Object> configBuilder = ImmutableMap.builder()
         .put("username", config.get("username").asText())
-        .put("jdbc_url", String.format("jdbc:postgresql://%s:%s/%s",
-            config.get("host").asText(),
-            config.get("port").asText(),
-            config.get("database").asText()))
+        .put("jdbc_url", jdbcUrl.toString())
         .put("schema", schema);
 
     if (config.has("password")) {

diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json
@@ -52,13 +52,20 @@
         "airbyte_secret": true,
         "order": 5
       },
+      "ssl": {
+        "title": "SSL Connection",
+        "description": "Encrypt data using SSL.",
+        "type": "boolean",
+        "default": false,
+        "order": 6
+      },
       "basic_normalization": {
         "title": "Basic Normalization",
         "type": "boolean",
         "default": true,
         "description": "Whether or not to normalize the data in the destination. See <a href=\"https://docs.airbyte.io/architecture/basic-normalization\">basic normalization</a> for more details.",
         "examples": [true, false],
-        "order": 6
+        "order": 7
       }
     }
   }

diff --git a/...ntegration/java/io/airbyte/integrations/destination/postgres/PostgresIntegrationTest.java b/...ntegration/java/io/airbyte/integrations/destination/postgres/PostgresIntegrationTest.java
@@ -60,6 +60,7 @@ protected JsonNode getConfig() {
         .put("schema", "public")
         .put("port", db.getFirstMappedPort())
         .put("database", db.getDatabaseName())
+        .put("ssl", false)
         .build());
   }
 
@@ -72,6 +73,7 @@ protected JsonNode getFailCheckConfig() {
         .put("schema", "public")
         .put("port", db.getFirstMappedPort())
         .put("database", db.getDatabaseName())
+        .put("ssl", false)
         .build());
   }
 

diff --git a/...source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/...source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java
@@ -83,12 +83,24 @@ public PostgresSource() {
 
   @Override
   public JsonNode toJdbcConfig(JsonNode config) {
+
+    List<String> additionalParameters = new ArrayList<>();
+
+    final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:postgresql://%s:%s/%s?",
+        config.get("host").asText(),
+        config.get("port").asText(),
+        config.get("database").asText()));
+
+    if (config.get("ssl").asBoolean()) {
+      additionalParameters.add("ssl=true");
+      additionalParameters.add("sslmode=require");
+    }
+
+    additionalParameters.forEach(x -> jdbcUrl.append(x).append("&"));
+
     final ImmutableMap.Builder<Object, Object> configBuilder = ImmutableMap.builder()
         .put("username", config.get("username").asText())
-        .put("jdbc_url", String.format("jdbc:postgresql://%s:%s/%s",
-            config.get("host").asText(),
-            config.get("port").asText(),
-            config.get("database").asText()));
+        .put("jdbc_url", jdbcUrl.toString());
 
     if (config.has("password")) {
       configBuilder.put("password", config.get("password").asText());

diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json
@@ -4,7 +4,7 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Postgres Source Spec",
     "type": "object",
-    "required": ["host", "port", "database", "username"],
+    "required": ["host", "port", "database", "username", "ssl"],
     "additionalProperties": false,
     "properties": {
       "host": {
@@ -42,11 +42,18 @@
         "airbyte_secret": true,
         "order": 4
       },
+      "ssl": {
+        "title": "Connect using SSL",
+        "description": "Encrypt client/server communications for increased security.",
+        "type": "boolean",
+        "default": false,
+        "order": 5
+      },
       "replication_method": {
         "type": "object",
         "title": "Replication Method",
         "description": "Replication method to use for extracting data from the database.",
-        "order": 5,
+        "order": 6,
         "oneOf": [
           {
             "title": "Standard",

diff --git a/...byte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceStandardTest.java b/...byte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceStandardTest.java
@@ -72,6 +72,7 @@ protected void setup(TestDestinationEnv testEnv) throws Exception {
         .put("database", container.getDatabaseName())
         .put("username", container.getUsername())
         .put("password", container.getPassword())
+        .put("ssl", false)
         .put("replication_method", ImmutableMap.of("replication_slot", SLOT_NAME_BASE))
         .build());
 

diff --git a/...airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStandardTest.java b/...airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStandardTest.java
@@ -65,6 +65,7 @@ protected void setup(TestDestinationEnv testEnv) throws Exception {
         .put("database", container.getDatabaseName())
         .put("username", container.getUsername())
         .put("password", container.getPassword())
+        .put("ssl", false)
         .build());
 
     final Database database = Databases.createDatabase(

diff --git a/...postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/...postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java
@@ -193,6 +193,7 @@ private JsonNode getConfig(PostgreSQLContainer<?> psqlDb, String dbName) {
         .put("database", dbName)
         .put("username", psqlDb.getUsername())
         .put("password", psqlDb.getPassword())
+        .put("ssl", false)
         .put("replication_method", replicationMethod)
         .build());
   }

diff --git a/...src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcStandardSourceTest.java b/...src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcStandardSourceTest.java
@@ -60,6 +60,7 @@ public void setup() throws Exception {
         .put("database", dbName)
         .put("username", PSQL_DB.getUsername())
         .put("password", PSQL_DB.getPassword())
+        .put("ssl", false)
         .build());
 
     final String initScriptName = "init_" + dbName.concat(".sql");