Source oauth0: new streams and fix incremental (airbytehq#29001)

* Add new streams Organizations,OrganizationMembers,OrganizationMemberRoles * relax schema definition to allow additional fields * Bump image tag version * revert some changes to the old schemas * Format python so gradle can pass * update incremental * remove unused print * fix unit test --------- Co-authored-by: Vasilis Gavriilidis <[email protected]>
Grayfados · Aug 3, 2023 · 03ffad5 · 03ffad5
1 parent 3bc79be
commit 03ffad5
Show file tree

Hide file tree

Showing 16 changed files with 415 additions and 52 deletions.
diff --git a/airbyte-integrations/connectors/source-auth0/Dockerfile b/airbyte-integrations/connectors/source-auth0/Dockerfile
@@ -34,5 +34,5 @@ COPY source_auth0 ./source_auth0
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.0
+LABEL io.airbyte.version=0.3.0
 LABEL io.airbyte.name=airbyte/source-auth0
diff --git a/airbyte-integrations/connectors/source-auth0/acceptance-test-config.yml b/airbyte-integrations/connectors/source-auth0/acceptance-test-config.yml
@@ -1,22 +1,30 @@
 connector_image: airbyte/source-auth0:dev
-tests:
+acceptance_tests:
   spec:
-    - spec_path: "source_auth0/spec.yaml"
+    tests:
+      - spec_path: "source_auth0/spec.yaml"
   connection:
-    - config_path: "secrets/config.json"
-      status: "succeed"
-    - config_path: "integration_tests/invalid_config.json"
-      status: "failed"
+    tests:
+      - config_path: "secrets/config.json"
+        status: "succeed"
+      - config_path: "integration_tests/invalid_config.json"
+        status: "failed"
   discovery:
-    - config_path: "secrets/config.json"
+    tests:
+      - config_path: "secrets/config.json"
   basic_read:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      empty_streams: []
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        empty_streams: []
+        fail_on_extra_columns: false
   incremental:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      future_state_path: "integration_tests/abnormal_state.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        future_state:
+          future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-auth0/integration_tests/abnormal_state.json
@@ -1,3 +1,9 @@
-{
-  "users": { "updated_at": "3021-09-08T07:04:28.000Z" }
-}
+[
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": { "updated_at": "5000-08-02T16:18:47.824Z" },
+      "stream_descriptor": { "name": "users" }
+    }
+  }
+]
diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-auth0/integration_tests/configured_catalog.json
@@ -20,6 +20,36 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite",
       "primary_key": [["client_id"]]
+    },
+    {
+      "stream": {
+        "name": "organizations",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite",
+      "primary_key": [["id"]]
+    },
+    {
+      "stream": {
+        "name": "organization_members",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite",
+      "primary_key": [["id"]]
+    },
+    {
+      "stream": {
+        "name": "organization_member_roles",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite",
+      "primary_key": [["id"]]
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_state.json
@@ -1,3 +1,9 @@
-{
-  "users": { "updated_at": "2021-09-08T07:04:28.000Z" }
-}
+[
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": { "updated_at": "2000-08-02T16:18:47.824Z" },
+      "stream_descriptor": { "name": "users" }
+    }
+  }
+]
diff --git a/airbyte-integrations/connectors/source-auth0/metadata.yaml b/airbyte-integrations/connectors/source-auth0/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 6c504e48-14aa-4221-9a72-19cf5ff1ae78
-  dockerImageTag: 0.2.0
+  dockerImageTag: 0.3.0
   dockerRepository: airbyte/source-auth0
   githubIssueLabel: source-auth0
   icon: auth0.svg

diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/clients.json b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/clients.json
@@ -121,7 +121,7 @@
       }
     },
     "signing_keys": {
-      "type": ["array", "null"],
+      "type": ["null", "array"],
       "items": {
         "type": ["object", "null"],
         "additionalProperties": true

diff --git a/...-integrations/connectors/source-auth0/source_auth0/schemas/organization_member_roles.json b/...-integrations/connectors/source-auth0/source_auth0/schemas/organization_member_roles.json
@@ -0,0 +1,22 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": ["object", "null"],
+  "additionalProperties": true,
+  "properties": {
+    "id": {
+      "type": ["string", "null"]
+    },
+    "org_id": {
+      "type": ["string", "null"]
+    },
+    "user_id": {
+      "type": ["string", "null"]
+    },
+    "name": {
+      "type": ["string", "null"]
+    },
+    "description": {
+      "type": ["string", "null"]
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organization_members.json b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organization_members.json
@@ -0,0 +1,25 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": ["object", "null"],
+    "additionalProperties": true,
+    "properties":{
+        "id": {
+          "type": ["string", "null"]
+        },
+        "org_id": {
+          "type": ["string", "null"]
+        },
+        "user_id": {
+          "type": ["string", "null"]
+        },
+        "name": {
+          "type": ["string", "null"]
+        },
+        "email": {
+          "type": ["string", "null"]
+        },
+        "picture": {
+          "type": ["string", "null"]
+        }
+    }
+}
diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organizations.json b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organizations.json
@@ -0,0 +1,24 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": ["object", "null"],
+  "additionalProperties": true,
+  "properties": {
+    "id": {
+      "type": ["string", "null"]
+    },
+    "name": {
+      "type": ["string", "null"]
+    },
+    "display_name": {
+      "type": ["string", "null"]
+    },
+    "branding": {
+      "type": ["object", "null"],
+      "additionalProperties": true
+    },
+    "metadata": {
+      "type": ["object", "null"],
+      "additionalProperties": true
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/users.json b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/users.json
@@ -32,10 +32,15 @@
       "type": ["string", "null"]
     },
     "identities": {
-      "type": "array",
+      "type": ["null", "array"],
       "items": {
         "type": ["object", "null"],
-        "additionalProperties": true
+        "additionalProperties": true,
+        "properties": {
+          "connection": {
+            "type": ["string", "null"]
+          }
+        }
       }
     },
     "app_metadata": {
@@ -56,8 +61,11 @@
       "type": ["string", "null"]
     },
     "multifactor": {
-      "type": ["object", "null"],
-      "additionalProperties": true
+      "type": ["null", "array"],
+      "additionalProperties": true,
+      "items": {
+        "type": ["string", "null"]
+      }
     },
     "last_ip": {
       "type": ["string", "null"]