🎉 Source Github: add reaction streams (airbytehq#5860)

* Source Github: add reaction streams

Co-authored-by: ykurochkin <[email protected]>

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json

@@ -2,7 +2,7 @@
   "sourceDefinitionId": "ef69ef6e-aa7f-4af1-a01d-ef775033524e",
   "name": "GitHub",
   "dockerRepository": "airbyte/source-github",
-  "dockerImageTag": "0.1.9",
+  "dockerImageTag": "0.1.10",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/github",
   "icon": "github.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -39,7 +39,7 @@
 - sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
   name: GitHub
   dockerRepository: airbyte/source-github
-  dockerImageTag: 0.1.9
+  dockerImageTag: 0.1.10
   documentationUrl: https://docs.airbyte.io/integrations/sources/github
   icon: github.svg
 - sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1

airbyte-integrations/connectors/source-github/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.9
+LABEL io.airbyte.version=0.1.10
 LABEL io.airbyte.name=airbyte/source-github

airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json

@@ -43,6 +43,16 @@
       "destination_sync_mode": "append",
       "cursor_field": ["updated_at"]
     },
+    {
+      "stream": {
+        "name": "commit_comment_reactions",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "commit_comments",
@@ -82,6 +92,16 @@
       "destination_sync_mode": "append",
       "cursor_field": ["created_at"]
     },
+    {
+      "stream": {
+        "name": "issue_comment_reactions",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "issue_events",
@@ -118,6 +138,16 @@
       "destination_sync_mode": "append",
       "cursor_field": ["updated_at"]
     },
+    {
+      "stream": {
+        "name": "issue_reactions",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "issues",
@@ -154,6 +184,16 @@
       "destination_sync_mode": "append",
       "cursor_field": ["updated_at"]
     },
+    {
+      "stream": {
+        "name": "pull_request_comment_reactions",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "pull_request_stats",

airbyte-integrations/connectors/source-github/source_github/schemas/commit_comment_reactions.json

@@ -0,0 +1,4 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "reaction.json"
+}

airbyte-integrations/connectors/source-github/source_github/schemas/issue_comment_reactions.json

@@ -0,0 +1,4 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "reaction.json"
+}

airbyte-integrations/connectors/source-github/source_github/schemas/issue_reactions.json

@@ -0,0 +1,4 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "reaction.json"
+}

airbyte-integrations/connectors/source-github/source_github/schemas/pull_request_comment_reactions.json

@@ -0,0 +1,4 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "reaction.json"
+}

airbyte-integrations/connectors/source-github/source_github/schemas/shared/reaction.json

@@ -0,0 +1,24 @@
+{
+  "type": ["null", "object"],
+  "properties": {
+    "id": {
+      "type": ["null", "integer"]
+    },
+    "node_id": {
+      "type": ["null", "string"]
+    },
+    "content": {
+      "type": ["null", "string"]
+    },
+    "created_at": {
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
+    "user": {
+      "$ref": "user.json"
+    },
+    "repository": {
+      "type": ["null", "string"]
+    }
+  }
+}

airbyte-integrations/connectors/source-github/source_github/source.py

@@ -37,15 +37,19 @@
     Branches,
     Collaborators,
     Comments,
+    CommitCommentReactions,
     CommitComments,
     Commits,
     Events,
+    IssueCommentReactions,
     IssueEvents,
     IssueLabels,
     IssueMilestones,
+    IssueReactions,
     Issues,
     Organizations,
     Projects,
+    PullRequestCommentReactions,
     PullRequests,
     PullRequestStats,
     Releases,
@@ -112,15 +116,19 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             Branches(**full_refresh_args),
             Collaborators(**full_refresh_args),
             Comments(**incremental_args),
+            CommitCommentReactions(**incremental_args),
             CommitComments(**incremental_args),
             Commits(**incremental_args),
             Events(**incremental_args),
+            IssueCommentReactions(**incremental_args),
             IssueEvents(**incremental_args),
             IssueLabels(**full_refresh_args),
             IssueMilestones(**incremental_args),
+            IssueReactions(**incremental_args),
             Issues(**incremental_args),
             Organizations(**organization_args),
             Projects(**incremental_args),
+            PullRequestCommentReactions(**incremental_args),
             PullRequestStats(**full_refresh_args),
             PullRequests(**incremental_args),
             Releases(**incremental_args),

airbyte-integrations/connectors/source-github/source_github/streams.py

@@ -24,7 +24,8 @@
 
 import os
 import time
-from abc import ABC
+from abc import ABC, abstractmethod
+from copy import deepcopy
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 from urllib import parse
 
@@ -276,7 +277,9 @@ def read_records(
 class IncrementalGithubStream(SemiIncrementalGithubStream):
     def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]:
         params = super().request_params(stream_state=stream_state, **kwargs)
-        params["since"] = self.get_starting_point(stream_state=stream_state, repository=stream_slice["repository"])
+        since_params = self.get_starting_point(stream_state=stream_state, repository=stream_slice["repository"])
+        if since_params:
+            params["since"] = since_params
         return params
 
 
@@ -686,3 +689,69 @@ class ReviewComments(IncrementalGithubStream):
 
     def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
         return f"repos/{stream_slice['repository']}/pulls/comments"
+
+
+# Reactions streams
+
+
+class ReactionStream(GithubStream, ABC):
+
+    parent_key = "id"
+
+    def __init__(self, **kwargs):
+        self._stream_kwargs = deepcopy(kwargs)
+        self._parent_stream = self.parent_entity(**kwargs)
+        kwargs.pop("start_date", None)
+        super().__init__(**kwargs)
+
+    @property
+    @abstractmethod
+    def parent_entity(self):
+        """
+        Specify the class of the parent stream for which receive reactions
+        """
+
+    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
+        parent_path = self._parent_stream.path(stream_slice=stream_slice, **kwargs)
+        return f"{parent_path}/{stream_slice[self.parent_key]}/reactions"
+
+    def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
+        for stream_slice in super().stream_slices(**kwargs):
+            for parent_record in self._parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice):
+                yield {self.parent_key: parent_record[self.parent_key], "repository": stream_slice["repository"]}
+
+    def request_headers(self, **kwargs) -> Mapping[str, Any]:
+        return {"Accept": "application/vnd.github.squirrel-girl-preview+json"}
+
+
+class CommitCommentReactions(ReactionStream):
+    """
+    API docs: https://docs.github.com/en/rest/reference/reactions#list-reactions-for-a-commit-comment
+    """
+
+    parent_entity = CommitComments
+
+
+class IssueCommentReactions(ReactionStream):
+    """
+    API docs: https://docs.github.com/en/rest/reference/reactions#list-reactions-for-an-issue-comment
+    """
+
+    parent_entity = Comments
+
+
+class IssueReactions(ReactionStream):
+    """
+    API docs: https://docs.github.com/en/rest/reference/reactions#list-reactions-for-an-issue
+    """
+
+    parent_entity = Issues
+    parent_key = "number"
+
+
+class PullRequestCommentReactions(ReactionStream):
+    """
+    API docs: https://docs.github.com/en/rest/reference/reactions#list-reactions-for-a-pull-request-review-comment
+    """
+
+    parent_entity = ReviewComments

docs/integrations/sources/github.md

@@ -13,7 +13,7 @@ This connector outputs the following full refresh streams:
 * [Collaborators](https://docs.github.com/en/rest/reference/repos#list-repository-collaborators)
 * [Issue labels](https://docs.github.com/en/free-pro-team@latest/rest/reference/issues#list-labels-for-a-repository)
 * [Organizations](https://docs.github.com/en/rest/reference/orgs#get-an-organization)
-* [Pull Request Stats](https://docs.github.com/en/rest/reference/pulls#get-a-pull-request)
+* [Pull request stats](https://docs.github.com/en/rest/reference/pulls#get-a-pull-request)
 * [Repositories](https://docs.github.com/en/rest/reference/repos#list-organization-repositories)
 * [Reviews](https://docs.github.com/en/rest/reference/pulls#list-reviews-for-a-pull-request)
 * [Tags](https://docs.github.com/en/rest/reference/repos#list-repository-tags)
@@ -24,15 +24,19 @@ This connector outputs the following incremental streams:
 
 * [Comments](https://docs.github.com/en/rest/reference/issues#list-issue-comments-for-a-repository)
 * [Commits](https://docs.github.com/en/rest/reference/issues#list-issue-comments-for-a-repository)
+* [Commit comment reactions](https://docs.github.com/en/rest/reference/reactions#list-reactions-for-a-commit-comment)
 * [Commit comments](https://docs.github.com/en/rest/reference/repos#list-commit-comments-for-a-repository)
 * [Events](https://docs.github.com/en/rest/reference/activity#list-repository-events)
 * [Issues](https://docs.github.com/en/rest/reference/issues#list-repository-issues)
+* [Issue comment reactions](https://docs.github.com/en/rest/reference/reactions#list-reactions-for-an-issue-comment)
 * [Issue events](https://docs.github.com/en/rest/reference/issues#list-issue-events-for-a-repository)
 * [Issue milestones](https://docs.github.com/en/rest/reference/issues#list-milestones)
+* [Issue reactions](https://docs.github.com/en/rest/reference/reactions#list-reactions-for-an-issue)
 * [Projects](https://docs.github.com/en/rest/reference/projects#list-repository-projects)
 * [Pull requests](https://docs.github.com/en/rest/reference/pulls#list-pull-requests)
+* [Pull request comment reactions](https://docs.github.com/en/rest/reference/reactions#list-reactions-for-a-pull-request-review-comment)
 * [Releases](https://docs.github.com/en/rest/reference/repos#list-releases)
-* [Review Comments](https://docs.github.com/en/rest/reference/pulls#list-review-comments-in-a-repository)
+* [Review comments](https://docs.github.com/en/rest/reference/pulls#list-review-comments-in-a-repository)
 * [Stargazers](https://docs.github.com/en/rest/reference/activity#list-stargazers)
 
 ### Notes
@@ -93,6 +97,7 @@ Your token should have at least the `repo` scope. Depending on which streams you
 
 | Version | Date       | Pull Request | Subject |
 | :------ | :--------  | :-----       | :------ |
+| 0.1.10  | 2021-09-09 | [5860](https://github.com/airbytehq/airbyte/pull/5860) | Add reaction streams |
 | 0.1.9   | 2021-09-02 | [5788](https://github.com/airbytehq/airbyte/pull/5788) | Handling empty repository, check method using RepositoryStats stream |
 | 0.1.8   | 2021-09-01 | [5757](https://github.com/airbytehq/airbyte/pull/5757) | Add more streams |
 | 0.1.7   | 2021-08-27 | [5696](https://github.com/airbytehq/airbyte/pull/5696) | Handle negative backoff values |