Source PostHog: Use account information for checking the connection (a…

…irbytehq#4692) * this should fix the check if no records in annotations stream * update schemas for new SAT requirements && apply user hint upgrade on wrong api key * save schema upd * upd insights schema * upd insights schema2 * upd insights schema3 * upd insights schema4 * upd insights schema5 (null is joking) * upd insights schema6 (null is joking) * upd insights schema7 * upd insights schema8 * upd insights schema8 * bump version && docs
beakalyk · Jul 15, 2021 · a10c805 · a10c805
1 parent a59e5f6
commit a10c805
Show file tree

Hide file tree

Showing 12 changed files with 65 additions and 51 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/af6d50ee-dddf-4126-a8ee-7faee990774f.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/af6d50ee-dddf-4126-a8ee-7faee990774f.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "af6d50ee-dddf-4126-a8ee-7faee990774f",
   "name": "PostHog",
   "dockerRepository": "airbyte/source-posthog",
-  "dockerImageTag": "0.1.1",
+  "dockerImageTag": "0.1.2",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/posthog"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -57,7 +57,7 @@
 - sourceDefinitionId: af6d50ee-dddf-4126-a8ee-7faee990774f
   name: PostHog
   dockerRepository: airbyte/source-posthog
-  dockerImageTag: 0.1.1
+  dockerImageTag: 0.1.2
   documentationUrl: https://docs.airbyte.io/integrations/sources/posthog
 - sourceDefinitionId: cd42861b-01fc-4658-a8ab-5d11d0510f01
   name: Recurly

diff --git a/airbyte-integrations/connectors/source-posthog/Dockerfile b/airbyte-integrations/connectors/source-posthog/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.version=0.1.2
 LABEL io.airbyte.name=airbyte/source-posthog
diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-posthog/integration_tests/configured_catalog.json
@@ -82,7 +82,6 @@
         "supported_sync_modes": ["full_refresh"],
         "source_defined_cursor": null,
         "default_cursor_field": null,
-        "source_defined_primary_key": [["id"]],
         "namespace": null
       },
       "sync_mode": "full_refresh",
@@ -97,7 +96,6 @@
         "supported_sync_modes": ["full_refresh"],
         "source_defined_cursor": null,
         "default_cursor_field": null,
-        "source_defined_primary_key": [["id"]],
         "namespace": null
       },
       "sync_mode": "full_refresh",
@@ -127,7 +125,6 @@
         "supported_sync_modes": ["full_refresh"],
         "source_defined_cursor": null,
         "default_cursor_field": null,
-        "source_defined_primary_key": [["id"]],
         "namespace": null
       },
       "sync_mode": "full_refresh",

diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/annotations.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/annotations.json
@@ -5,17 +5,17 @@
       "type": "integer"
     },
     "content": {
-      "type": "string"
+      "type": ["string", "null"]
     },
     "date_marker": {
-      "type": "string",
+      "type": ["string", "null"],
       "format": "date-time"
     },
     "creation_type": {
-      "type": "string"
+      "type": ["string", "null"]
     },
     "dashboard_item": {
-      "type": "string"
+      "type": ["string", "null"]
     },
     "created_by": {
       "type": "object",

diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/cohorts.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/cohorts.json
@@ -8,18 +8,18 @@
       "type": "string"
     },
     "groups": {
-      "type": "array",
+      "type": ["array", "object"],
       "items": {
         "type": "object",
         "properties": {
           "days": {
-            "type": "string"
+            "type": ["string", "null"]
           },
           "action_id": {
-            "type": "string"
+            "type": ["string", "null"]
           },
           "properties": {
-            "type": "array",
+            "type": ["array"],
             "items": {
               "type": "object"
             }
@@ -46,19 +46,19 @@
           "type": "string"
         },
         "first_name": {
-          "type": "string"
+          "type": ["string", "null"]
         },
         "email": {
-          "type": "string"
+          "type": ["string", "null"]
         }
       }
     },
     "created_at": {
-      "type": "string",
+      "type": ["string", "null"],
       "format": "date-time"
     },
     "last_calculation": {
-      "type": "string"
+      "type": ["string", "null"]
     },
     "errors_calculating": {
       "type": "integer"

diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events.json
@@ -8,7 +8,7 @@
       "type": "string"
     },
     "event": {
-      "type": "string"
+      "type": ["string", "object"]
     },
     "timestamp": {
       "type": "string",
@@ -34,7 +34,7 @@
     "elements": {
       "type": "array",
       "items": {
-        "type": "string"
+        "type": ["string", "object"]
       }
     },
     "elements_chain": {

diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/feature_flags.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/feature_flags.json
@@ -11,7 +11,7 @@
       "type": "string"
     },
     "rollout_percentage": {
-      "type": "integer"
+      "type": ["integer", "null"]
     },
     "filters": {
       "type": "object",

diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json
@@ -5,118 +5,118 @@
       "type": "integer"
     },
     "name": {
-      "type": "string"
+      "type": ["string", "null"]
     },
     "filters": {
-      "type": "object",
+      "type": ["object", "null"],
       "properties": {
         "events": {
           "type": "array",
           "items": {
             "type": "object",
             "properties": {
               "id": {
-                "type": "string"
+                "type": ["string", "null"]
               },
               "math": {
-                "type": "string"
+                "type": ["string", "null"]
               },
               "name": {
-                "type": "string"
+                "type": ["string", "null"]
               },
               "type": {
-                "type": "string"
+                "type": ["string", "null"]
               },
               "order": {
-                "type": "integer"
+                "type": ["integer", "null"]
               },
               "properties": {
                 "type": "array",
                 "items": {
-                  "type": "string"
+                  "type": ["string", "null", "object"]
                 }
               },
               "math_property": {
-                "type": "string"
+                "type": ["string", "null"]
               }
             }
           }
         },
         "display": {
-          "type": "string"
+          "type": ["string", "null"]
         },
         "filters": {
           "type": "array",
           "items": {
-            "type": "string"
+            "type": ["string", "null"]
           }
         },
         "insight": {
-          "type": "string"
+          "type": ["string", "null"]
         },
         "session": {
-          "type": "string"
+          "type": ["string", "null"]
         },
         "interval": {
-          "type": "string"
+          "type": ["string", "integer"]
         },
         "pagination": {
-          "type": "object"
+          "type": ["object", "null"]
         }
       }
     },
     "filters_hash": {
-      "type": "string"
+      "type": ["string", "null"]
     },
     "order": {
-      "type": "string"
+      "type": ["string", "null", "object"]
     },
     "deleted": {
       "type": "boolean"
     },
     "dashboard": {
-      "type": "string"
+      "type": ["string", "integer", "null"]
     },
     "layouts": {
-      "type": "object"
+      "type": ["object", "null"]
     },
     "color": {
-      "type": "string"
+      "type": ["string", "null"]
     },
     "last_refresh": {
-      "type": "string",
+      "type": ["string", "null"],
       "format": "date-time"
     },
     "refreshing": {
       "type": "boolean"
     },
     "result": {
-      "type": "string"
+      "type": ["string", "null", "object"]
     },
     "created_at": {
-      "type": "string",
+      "type": ["string", "null"],
       "format": "date-time"
     },
     "saved": {
       "type": "boolean"
     },
     "created_by": {
-      "type": "object",
+      "type": ["object", "null"],
       "properties": {
         "id": {
           "type": "integer"
         },
         "uuid": {
-          "type": "string"
+          "type": ["string", "null"]
         },
         "distinct_id": {
-          "type": "string"
+          "type": ["string", "null"]
         },
         "first_name": {
-          "type": "string"
+          "type": ["string", "null"]
         },
         "email": {
-          "type": "string"
+          "type": ["string", "null"]
         }
       }
     }

diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/source.py b/airbyte-integrations/connectors/source-posthog/source_posthog/source.py
@@ -22,10 +22,10 @@
 # SOFTWARE.
 #
 
-
 from typing import Any, List, Mapping, Tuple
 
 import pendulum
+import requests
 from airbyte_cdk.logger import AirbyteLogger
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources import AbstractSource
@@ -41,6 +41,7 @@
     InsightsPath,
     InsightsSessions,
     Persons,
+    PingMe,
     Trends,
 )
 
@@ -50,11 +51,13 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
         try:
             _ = pendulum.parse(config["start_date"], strict=True)
             authenticator = TokenAuthenticator(token=config["api_key"])
-            stream = Cohorts(authenticator=authenticator)
+            stream = PingMe(authenticator=authenticator)
             records = stream.read_records(sync_mode=SyncMode.full_refresh)
             _ = next(records)
             return True, None
         except Exception as e:
+            if isinstance(e, requests.exceptions.HTTPError) and e.response.status_code == 401:
+                return False, f"Please check you api_key. Error: {repr(e)}"
             return False, repr(e)
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:

diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py b/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py
@@ -228,3 +228,16 @@ class Trends(PosthogStream):
 
     def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
         return "insight/trend"
+
+
+class PingMe(PosthogStream):
+    """
+    Docs: https://posthog.com/docs/api/user
+    """
+
+    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
+        return "users/@me"
+
+    def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]:
+        response_json = response.json()
+        yield response_json
diff --git a/docs/integrations/sources/posthog.md b/docs/integrations/sources/posthog.md
@@ -56,5 +56,6 @@ Please follow these [steps](https://posthog.com/docs/api/overview#how-to-obtain-
 
 | Version | Date       | Pull Request | Subject |
 | :------ | :--------  | :-----       | :------ |
+| 0.1.2   | 2021-07-15 | [4692](https://github.com/airbytehq/airbyte/pull/4692) | Source PostHog: Use account information for checking the connection
 | 0.1.1   | 2021-07-05 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add `AIRBYTE_ENTRYPOINT` env variable for kubernetes support|
 | 0.1.0   | 2021-06-08 | [3768](https://github.com/airbytehq/airbyte/pull/3768) | Initial Release |