📝 🎉 Source Dixa: Migrated to CI Sandbox, refactored code structure fo…

…r future support (airbytehq#5367)
soufianeodf · Aug 18, 2021 · 0f55bf6 · 0f55bf6
1 parent 729dacd
commit 0f55bf6
Show file tree

Hide file tree

Showing 21 changed files with 206 additions and 207 deletions.
diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml
@@ -88,6 +88,7 @@ jobs:
           DESTINATION_KEEN_TEST_CREDS: ${{ secrets.DESTINATION_KEEN_TEST_CREDS }}
           DESTINATION_KVDB_TEST_CREDS: ${{ secrets.DESTINATION_KVDB_TEST_CREDS }}
           DRIFT_INTEGRATION_TEST_CREDS: ${{ secrets.DRIFT_INTEGRATION_TEST_CREDS }}
+          SOURCE_DIXA_TEST_CREDS: ${{ secrets.SOURCE_DIXA_TEST_CREDS }}
           EXCHANGE_RATES_TEST_CREDS: ${{ secrets.EXCHANGE_RATES_TEST_CREDS }}
           FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }}
           FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS }}

diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml
@@ -88,6 +88,7 @@ jobs:
           DESTINATION_KEEN_TEST_CREDS: ${{ secrets.DESTINATION_KEEN_TEST_CREDS }}
           DESTINATION_KVDB_TEST_CREDS: ${{ secrets.DESTINATION_KVDB_TEST_CREDS }}
           DRIFT_INTEGRATION_TEST_CREDS: ${{ secrets.DRIFT_INTEGRATION_TEST_CREDS }}
+          SOURCE_DIXA_TEST_CREDS: ${{ secrets.SOURCE_DIXA_TEST_CREDS }}
           EXCHANGE_RATES_TEST_CREDS: ${{ secrets.EXCHANGE_RATES_TEST_CREDS }}
           FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }}
           FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS }}

diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/0b5c867e-1b12-4d02-ab74-97b2184ff6d7.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/0b5c867e-1b12-4d02-ab74-97b2184ff6d7.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "0b5c867e-1b12-4d02-ab74-97b2184ff6d7",
   "name": "Dixa",
   "dockerRepository": "airbyte/source-dixa",
-  "dockerImageTag": "0.1.0",
+  "dockerImageTag": "0.1.1",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/dixa"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -385,7 +385,7 @@
 - sourceDefinitionId: 0b5c867e-1b12-4d02-ab74-97b2184ff6d7
   name: Dixa
   dockerRepository: airbyte/source-dixa
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.1.1
   documentationUrl: https://docs.airbyte.io/integrations/sources/dixa
 - sourceDefinitionId: e7eff203-90bf-43e5-a240-19ea3056c474
   name: Typeform

diff --git a/airbyte-integrations/connectors/source-dixa/Dockerfile b/airbyte-integrations/connectors/source-dixa/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.1.1
 LABEL io.airbyte.name=airbyte/source-dixa
diff --git a/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml b/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml
@@ -14,20 +14,13 @@ tests:
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file
-#      expect_records:
-#        path: "integration_tests/expected_records.txt"
-#        extra_fields: no
-#        exact_order: no
-#        extra_records: yes
+      timeout_seconds: 1200
   incremental:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-#      future_state_path: "integration_tests/abnormal_state.json"
-# We skip the full_refresh test because of unusual behaviour in the Dixa API.
-# We observed cases where a record was updated without the updated_at value changing.
-# See the thread below for further information:
-# https://airbytehq.slack.com/archives/C01VDDEGL7M/p1625319909273300
-#  full_refresh:
-#    - config_path: "secrets/config.json"
-#      configured_catalog_path: "integration_tests/configured_catalog.json"
+      future_state_path: "integration_tests/abnormal_state.json"
+      timeout_seconds: 1200
+  full_refresh:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      timeout_seconds: 1200
diff --git a/airbyte-integrations/connectors/source-dixa/build.gradle b/airbyte-integrations/connectors/source-dixa/build.gradle
@@ -1,8 +1,7 @@
 plugins {
     id 'airbyte-python'
     id 'airbyte-docker'
-    // TODO acceptance tests are disabled in CI pending a Dixa Sandbox: https://github.com/airbytehq/airbyte/issues/4667
-//    id 'airbyte-source-acceptance-test'
+    id 'airbyte-source-acceptance-test'
 }
 
 airbytePython {

diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-dixa/integration_tests/abnormal_state.json
@@ -1,5 +1,5 @@
 {
   "conversation_export": {
-    "updated_at": 9999999999999
+    "updated_at": 4778168400000
   }
 }
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py
@@ -30,7 +30,4 @@
 
 @pytest.fixture(scope="session", autouse=True)
 def connector_setup():
-    """ This fixture is a placeholder for external resources that acceptance test might require."""
-    # TODO: setup test dependencies if needed. otherwise remove the TODO comments
     yield
-    # TODO: clean up test dependencies
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/catalog.json b/airbyte-integrations/connectors/source-dixa/integration_tests/catalog.json
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-dixa/integration_tests/configured_catalog.json
@@ -1,17 +1,17 @@
 {
   "streams": [
     {
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append",
-      "cursor_field": ["updated_at"],
       "stream": {
         "name": "conversation_export",
+        "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
         "default_cursor_field": ["updated_at"],
-        "source_defined_primary_key": [["id"]],
-        "json_schema": {}
-      }
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["updated_at"],
+      "destination_sync_mode": "append"
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-dixa/integration_tests/invalid_config.json
@@ -1,4 +1,5 @@
 {
-  "start_timestamp": 1625263200000,
-  "batch_size": 1
+  "api_token": "TOKEN",
+  "start_date": "2020-01-01",
+  "batch_size": "31"
 }
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-dixa/integration_tests/sample_config.json
@@ -1,3 +1,5 @@
 {
-  "fix-me": "TODO"
+  "api_token": "TOKEN",
+  "start_date": "2020-01-01",
+  "batch_size": "31"
 }
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-dixa/integration_tests/sample_state.json
@@ -1,5 +1,5 @@
 {
-  "todo-stream-name": {
-    "todo-field-name": "value"
+  "conversation_export": {
+    "updated_at": 1628748165633
   }
 }
diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json b/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json
@@ -1,18 +1,18 @@
 {
-  "type": "object",
+  "type": ["null", "object"],
   "required": ["id", "created_at", "initial_channel", "requester_id"],
   "properties": {
     "id": {
-      "type": "integer"
+      "type": ["null", "integer"]
     },
     "created_at": {
-      "type": "integer"
+      "type": ["null", "integer"]
     },
     "initial_channel": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "requester_id": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "requester_name": {
       "type": ["null", "string"]
@@ -42,9 +42,9 @@
       "type": ["null", "string"]
     },
     "ratings": {
-      "type": "array",
+      "type": ["null", "array"],
       "items": {
-        "type": "object",
+        "type": ["null", "object"],
         "properties": {
           "rating_score": {
             "type": ["null", "integer"]
@@ -142,13 +142,13 @@
     "tags": {
       "type": ["null", "array"],
       "items": {
-        "type": "string"
+        "type": ["null", "string"]
       }
     },
     "conversation_wrapup_notes": {
       "type": ["null", "array"],
       "items": {
-        "type": "string"
+        "type": ["null", "string"]
       }
     },
     "transferee_name": {