From 6adad7d98e5a08a83495f7438642e2fee1e88908 Mon Sep 17 00:00:00 2001 From: Subodh Kant Chaturvedi Date: Tue, 1 Jun 2021 17:40:28 +0530 Subject: [PATCH] destination-specification: add supportsNormalization and supportsDBT attributes (#3779) * destination-specification: add supportsNormalization and supportsDBT attributes * address review comment * missed this one * output after gradle format --- airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py | 2 ++ .../airbyte_protocol/models/airbyte_protocol.py | 2 ++ .../destination-bigquery/src/main/resources/spec.json | 2 ++ .../destination-csv/src/main/resources/spec.json | 2 ++ .../destination-e2e-test/src/main/resources/spec.json | 2 ++ .../destination-jdbc/src/main/resources/spec.json | 2 ++ .../destination-local-json/src/main/resources/spec.json | 2 ++ .../destination-meilisearch/src/main/resources/spec.json | 2 ++ .../destination-mssql/src/main/resources/spec.json | 2 ++ .../destination-mysql/src/main/resources/spec.json | 2 ++ .../destination-postgres/src/main/resources/spec.json | 2 ++ .../destination-redshift/src/main/resources/spec.json | 2 ++ .../destination-snowflake/src/main/resources/spec.json | 2 ++ .../main/resources/airbyte_protocol/airbyte_protocol.yaml | 7 +++++++ 14 files changed, 33 insertions(+) diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py index 6efd61b6d3c3..11b73fcb3f6d 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py @@ -114,6 +114,8 @@ class Config: description="ConnectorDefinition specific blob. Must be a valid JSON string.", ) supportsIncremental: Optional[bool] = Field(None, description="If the connector supports incremental mode or not.") + supportsNormalization: Optional[bool] = Field(None, description="If the connector supports normalization or not.") + supportsDBT: Optional[bool] = Field(None, description="If the connector supports DBT or not.") supported_destination_sync_modes: Optional[List[DestinationSyncMode]] = Field( None, description="List of destination sync modes supported by the connector" ) diff --git a/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py b/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py index 6efd61b6d3c3..11b73fcb3f6d 100644 --- a/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py +++ b/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py @@ -114,6 +114,8 @@ class Config: description="ConnectorDefinition specific blob. Must be a valid JSON string.", ) supportsIncremental: Optional[bool] = Field(None, description="If the connector supports incremental mode or not.") + supportsNormalization: Optional[bool] = Field(None, description="If the connector supports normalization or not.") + supportsDBT: Optional[bool] = Field(None, description="If the connector supports DBT or not.") supported_destination_sync_modes: Optional[List[DestinationSyncMode]] = Field( None, description="List of destination sync modes supported by the connector" ) diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json index 8249fe1be10f..98a97b61009b 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json @@ -1,6 +1,8 @@ { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", "supportsIncremental": true, + "supportsNormalization": true, + "supportsDBT": true, "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-csv/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-csv/src/main/resources/spec.json index 737572addd6b..389af6735425 100644 --- a/airbyte-integrations/connectors/destination-csv/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-csv/src/main/resources/spec.json @@ -1,6 +1,8 @@ { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/local-csv", "supportsIncremental": true, + "supportsNormalization": false, + "supportsDBT": false, "supported_destination_sync_modes": ["overwrite", "append"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json index e61452e0ade3..ea4ba00390bf 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json @@ -1,6 +1,8 @@ { "documentationUrl": "https://example.com", "supportsIncremental": true, + "supportsNormalization": false, + "supportsDBT": false, "supported_destination_sync_modes": ["overwrite", "append"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-jdbc/src/main/resources/spec.json index 5b8ff21c0a35..9991afa2356c 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/resources/spec.json @@ -1,6 +1,8 @@ { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/postgres", "supportsIncremental": true, + "supportsNormalization": false, + "supportsDBT": false, "supported_destination_sync_modes": ["overwrite", "append"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-local-json/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-local-json/src/main/resources/spec.json index 7f46c3836835..3acd992f190a 100644 --- a/airbyte-integrations/connectors/destination-local-json/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-local-json/src/main/resources/spec.json @@ -1,6 +1,8 @@ { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/local-json", "supportsIncremental": true, + "supportsNormalization": false, + "supportsDBT": false, "supported_destination_sync_modes": ["overwrite", "append"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-meilisearch/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-meilisearch/src/main/resources/spec.json index 13bb56c4110b..34e6427769c2 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-meilisearch/src/main/resources/spec.json @@ -1,6 +1,8 @@ { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/meilisearch", "supportsIncremental": true, + "supportsNormalization": false, + "supportsDBT": false, "supported_destination_sync_modes": ["overwrite", "append"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-mssql/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-mssql/src/main/resources/spec.json index 48284392671a..af7641926175 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-mssql/src/main/resources/spec.json @@ -1,6 +1,8 @@ { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/postgres", "supportsIncremental": true, + "supportsNormalization": false, + "supportsDBT": true, "supported_destination_sync_modes": ["overwrite", "append"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json index 37aca0a581c4..72aeb904cb61 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json @@ -1,6 +1,8 @@ { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mysql", "supportsIncremental": true, + "supportsNormalization": false, + "supportsDBT": true, "supported_destination_sync_modes": ["overwrite", "append"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json index f73ec3106059..0594bd9da894 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json @@ -1,6 +1,8 @@ { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/postgres", "supportsIncremental": true, + "supportsNormalization": true, + "supportsDBT": true, "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json index ceaeae16b6a3..9c4b855c1a94 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json @@ -1,6 +1,8 @@ { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift", "supportsIncremental": true, + "supportsNormalization": true, + "supportsDBT": true, "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json index 577392217371..bc1be51fd28f 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json @@ -1,6 +1,8 @@ { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake", "supportsIncremental": true, + "supportsNormalization": true, + "supportsDBT": true, "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index cdbafe2426cf..4d399492c0b0 100644 --- a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -236,6 +236,13 @@ definitions: description: If the connector supports incremental mode or not. type: boolean # Destination Connectors Properties + #Normalization is currently implemented using dbt so it requires `supportsDBT` to be true for this to be true. + supportsNormalization: + description: If the connector supports normalization or not. + type: boolean + supportsDBT: + description: If the connector supports DBT or not. + type: boolean supported_destination_sync_modes: description: List of destination sync modes supported by the connector type: array