Source Jira: add options to stream (small fixes) (airbytehq#8141)

* Add option to render fields in HTML format and fix sprint_issue ids * Bump connector version * run format * run format * run seed generaotr * run format Co-authored-by: Chris Wu <[email protected]>
jabari-teampay · Nov 19, 2021 · dcc72ba · dcc72ba
1 parent 15b8501
commit dcc72ba
Show file tree

Hide file tree

Showing 11 changed files with 41 additions and 31 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -2787,7 +2787,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-jira:0.2.14"
+- dockerImage: "airbyte/source-jira:0.2.15"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/jira"
     connectionSpecification:
@@ -2849,6 +2849,12 @@
           title: "Expand Issue Changelog"
           description: "Expand the changelog when replicating issues"
           default: false
+        render_fields:
+          type: "boolean"
+          title: "Render Issue Fields"
+          description: "Render issue fields in HTML format in addition to Jira's JSON-like\
+            \ format"
+          default: false
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []

diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/CloudLogs.java b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/CloudLogs.java
@@ -63,8 +63,8 @@ static CloudLogs createCloudLogClient(final LogConfigs configs) {
   }
 
   /**
-   * Logs are configured to go to a Minio instance if the S3MinioEndpoint configuration specifically is set, along with the
-   * other default S3 logging configurations.
+   * Logs are configured to go to a Minio instance if the S3MinioEndpoint configuration specifically
+   * is set, along with the other default S3 logging configurations.
    *
    * @param configs contains the environment variables to check
    * @return boolean indicating if logs are configured to be sent to minio
@@ -75,8 +75,8 @@ private static boolean hasMinioConfiguration(final LogConfigs configs) {
   }
 
   /**
-   * Logs are configured to go to S3 if the S3LogBucketRegion configuration specifically is set, along with the
-   * other default S3 logging configurations.
+   * Logs are configured to go to S3 if the S3LogBucketRegion configuration specifically is set, along
+   * with the other default S3 logging configurations.
    *
    * @param configs contains the configuration values to check
    * @return boolean indicating if logs are configured to be sent to S3

diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py b/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py
@@ -12,6 +12,7 @@ class HarvestMixin:
     Mixin class for providing additional HTTP header for specifying account ID
     https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/
     """
+
     def __init__(self, *, account_id: str, account_id_header: str = "Harvest-Account-ID", **kwargs):
         super().__init__(**kwargs)
         self.account_id = account_id

diff --git a/...ors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/...ors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java
@@ -170,7 +170,7 @@ private Predicate<JsonNode> excludeNotAccessibleTables(Set<String> internalSchem
       return tablesWithSelectGrantPrivilege.stream()
           .anyMatch(e -> e.getSchemaName().equals(jsonNode.get(INTERNAL_SCHEMA_NAME).asText()))
           && tablesWithSelectGrantPrivilege.stream()
-          .anyMatch(e -> e.getTableName().equals(jsonNode.get(INTERNAL_TABLE_NAME).asText()))
+              .anyMatch(e -> e.getTableName().equals(jsonNode.get(INTERNAL_TABLE_NAME).asText()))
           && !internalSchemas.contains(jsonNode.get(INTERNAL_SCHEMA_NAME).asText());
     };
   }

diff --git a/...s/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/dto/JdbcPrivilegeDto.java b/...s/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/dto/JdbcPrivilegeDto.java
@@ -1,3 +1,7 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
 package io.airbyte.integrations.source.jdbc.dto;
 
 import com.google.common.base.Objects;
@@ -69,6 +73,7 @@ public JdbcPrivilegeDtoBuilder privilege(String privilege) {
     public JdbcPrivilegeDto build() {
       return new JdbcPrivilegeDto(grantee, tableName, schemaName, privilege);
     }
+
   }
 
   @Override
@@ -98,4 +103,5 @@ public String toString() {
         ", privilege='" + privilege + '\'' +
         '}';
   }
+
 }
diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py
@@ -349,7 +349,7 @@ def request_params(
         if self._render_fields:
             expand.append("renderedFields")
         if expand:
-            params["expand"] = ','.join(expand)
+            params["expand"] = ",".join(expand)
         return params
 
     def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]:

diff --git a/...source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/...source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java
@@ -237,19 +237,17 @@ private static AirbyteStream removeIncrementalWithoutPk(final AirbyteStream stre
   @Override
   public Set<JdbcPrivilegeDto> getPrivilegesTableForCurrentUser(JdbcDatabase database, String schema) throws SQLException {
     return database.bufferedResultSetQuery(
-            conn -> conn.getMetaData().getTablePrivileges(getCatalog(database), schema, null),
-            resultSet ->
-                JdbcPrivilegeDto.builder()
-                    .grantee(ofNullable(resultSet.getString(GRANTEE)).orElse(EMPTY))
-                    // we need the schema as different schemas could have tables with the same names
-                    .schemaName(ofNullable(resultSet.getString(JDBC_COLUMN_SCHEMA_NAME)).orElse(EMPTY))
-                    .tableName(ofNullable(resultSet.getString(JDBC_COLUMN_TABLE_NAME)).orElse(EMPTY))
-                    .privilege(ofNullable(resultSet.getString(PRIVILEGE)).orElse(EMPTY))
-                    .build())
+        conn -> conn.getMetaData().getTablePrivileges(getCatalog(database), schema, null),
+        resultSet -> JdbcPrivilegeDto.builder()
+            .grantee(ofNullable(resultSet.getString(GRANTEE)).orElse(EMPTY))
+            // we need the schema as different schemas could have tables with the same names
+            .schemaName(ofNullable(resultSet.getString(JDBC_COLUMN_SCHEMA_NAME)).orElse(EMPTY))
+            .tableName(ofNullable(resultSet.getString(JDBC_COLUMN_TABLE_NAME)).orElse(EMPTY))
+            .privilege(ofNullable(resultSet.getString(PRIVILEGE)).orElse(EMPTY))
+            .build())
         .stream()
-        .filter(jdbcPrivilegeDto ->
-            jdbcPrivilegeDto.getGrantee().equals(database.getDatabaseConfig().get("username").asText())
-                && "SELECT".equals(jdbcPrivilegeDto.getPrivilege()))
+        .filter(jdbcPrivilegeDto -> jdbcPrivilegeDto.getGrantee().equals(database.getDatabaseConfig().get("username").asText())
+            && "SELECT".equals(jdbcPrivilegeDto.getPrivilege()))
         .collect(toSet());
   }
 

diff --git a/...postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java b/...postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java
@@ -24,8 +24,6 @@
 import io.airbyte.db.Databases;
 import io.airbyte.protocol.models.AirbyteCatalog;
 import io.airbyte.protocol.models.AirbyteMessage;
-import io.airbyte.protocol.models.AirbyteMessage.Type;
-import io.airbyte.protocol.models.AirbyteRecordMessage;
 import io.airbyte.protocol.models.AirbyteStream;
 import io.airbyte.protocol.models.CatalogHelpers;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
@@ -34,9 +32,7 @@
 import io.airbyte.protocol.models.SyncMode;
 import io.airbyte.test.utils.PostgreSQLContainerHelper;
 import java.math.BigDecimal;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -186,4 +182,5 @@ void testIsCdc() {
         "publication", "ab_pub")));
     assertTrue(PostgresSource.isCdc(config));
   }
+
 }
diff --git a/...ce-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java b/...ce-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java
@@ -24,8 +24,6 @@
 import io.airbyte.db.Databases;
 import io.airbyte.protocol.models.AirbyteCatalog;
 import io.airbyte.protocol.models.AirbyteMessage;
-import io.airbyte.protocol.models.AirbyteMessage.Type;
-import io.airbyte.protocol.models.AirbyteRecordMessage;
 import io.airbyte.protocol.models.AirbyteStream;
 import io.airbyte.protocol.models.CatalogHelpers;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
@@ -34,9 +32,7 @@
 import io.airbyte.protocol.models.SyncMode;
 import io.airbyte.test.utils.PostgreSQLContainerHelper;
 import java.math.BigDecimal;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -229,4 +225,5 @@ void testIsCdc() {
         "publication", "ab_pub")));
     assertTrue(PostgresSource.isCdc(config));
   }
+
 }
diff --git a/...es/src/test/java/io/airbyte/integrations/source/postgres/utils/PostgresUnitTestsUtil.java b/...es/src/test/java/io/airbyte/integrations/source/postgres/utils/PostgresUnitTestsUtil.java
@@ -1,3 +1,7 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
 package io.airbyte.integrations.source.postgres.utils;
 
 import io.airbyte.commons.json.Jsons;
@@ -42,4 +46,5 @@ public static Map<Object, Object> map(final Object... entries) {
 
     };
   }
+
 }
diff --git a/...tional-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java b/...tional-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java
@@ -361,10 +361,10 @@ protected void assertColumnsWithSameNameAreSame(final String nameSpace, final St
   /**
    * @param database - The database where from privileges for tables will be consumed
    * @param schema - The schema where from privileges for tables will be consumed
-   * @return Set with privileges for tables for current DB-session user
-   * The method is responsible for SELECT-ing the table with privileges.
-   * In some cases such SELECT doesn't require (e.g. in Oracle DB - the schema is the user,
-   * you cannot REVOKE a privilege on a table from its owner).
+   * @return Set with privileges for tables for current DB-session user The method is responsible for
+   *         SELECT-ing the table with privileges. In some cases such SELECT doesn't require (e.g. in
+   *         Oracle DB - the schema is the user, you cannot REVOKE a privilege on a table from its
+   *         owner).
    * @throws SQLException
    */
   public <T> Set<T> getPrivilegesTableForCurrentUser(JdbcDatabase database, String schema) throws SQLException {