Source Amazon Seller Partner: GET_SELLER_FEEDBACK_DATA normalize head…

…er field names (airbytehq#9212)
soufianeodf · Jan 3, 2022 · 82d6b16 · 82d6b16
1 parent f9eac08
commit 82d6b16
Show file tree

Hide file tree

Showing 8 changed files with 34 additions and 16 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/e55879a8-0ef8-4557-abcf-ab34c53ec460.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/e55879a8-0ef8-4557-abcf-ab34c53ec460.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "e55879a8-0ef8-4557-abcf-ab34c53ec460",
   "name": "Amazon Seller Partner",
   "dockerRepository": "airbyte/source-amazon-seller-partner",
-  "dockerImageTag": "0.2.8",
+  "dockerImageTag": "0.2.9",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-seller-partner",
   "icon": "amazonsellerpartner.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -22,7 +22,7 @@
 - name: Amazon Seller Partner
   sourceDefinitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460
   dockerRepository: airbyte/source-amazon-seller-partner
-  dockerImageTag: 0.2.8
+  dockerImageTag: 0.2.9
   sourceType: api
   documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-seller-partner
   icon: amazonsellerpartner.svg

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -157,7 +157,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-amazon-seller-partner:0.2.8"
+- dockerImage: "airbyte/source-amazon-seller-partner:0.2.9"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/amazon-seller-partner"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/amazon-seller-partner"

diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile b/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.8
+LABEL io.airbyte.version=0.2.9
 LABEL io.airbyte.name=airbyte/source-amazon-seller-partner
diff --git a/...-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SELLER_FEEDBACK_DATA.json b/...-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SELLER_FEEDBACK_DATA.json
@@ -4,23 +4,23 @@
   "type": "object",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "properties": {
-    "Date": {
+    "date": {
       "type": ["null", "string"],
       "format": "date"
     },
-    "Rating": {
+    "rating": {
       "type": ["null", "number"]
     },
-    "Comments": {
+    "comments": {
       "type": ["null", "string"]
     },
-    "Response": {
+    "response": {
       "type": ["null", "string"]
     },
-    "Order ID": {
+    "order_id": {
       "type": ["null", "string"]
     },
-    "Rater Email": {
+    "rater_email": {
       "type": ["null", "string"]
     }
   }

diff --git a/...egrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/...egrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py
@@ -522,7 +522,7 @@ class SellerFeedbackReports(IncrementalReportsAmazonSPStream):
     MARKETPLACE_DATE_FORMAT_MAP = dict(
         # eu
         A2VIGQ35RCS4UG="D/M/YY",  # AE
-        A1PA6795UKMFR9="D/M/YY",  # DE
+        A1PA6795UKMFR9="D.M.YY",  # DE
         A1C3SOZRARQ6R3="D/M/YY",  # PL
         ARBP9OOSHTCHU="D/M/YY",  # EG
         A1RKKUPIHCS9HS="D/M/YY",  # ES
@@ -531,7 +531,7 @@ class SellerFeedbackReports(IncrementalReportsAmazonSPStream):
         APJ6JRA9NG5V4="D/M/YY",  # IT
         A1805IZSGTT6HS="D/M/YY",  # NL
         A17E79C6D8DWNP="D/M/YY",  # SA
-        A2NODRKZP88ZB9="D/M/YY",  # SE
+        A2NODRKZP88ZB9="YYYY-MM-DD",  # SE
         A33AVAJ2PDY3EV="D/M/YY",  # TR
         A1F83G8C2ARO7P="D/M/YY",  # UK
         # fe
@@ -541,12 +541,14 @@ class SellerFeedbackReports(IncrementalReportsAmazonSPStream):
         # na
         ATVPDKIKX0DER="M/D/YY",  # US
         A2Q3Y263D00KWC="D/M/YY",  # BR
-        A2EUQ1WTGCTBG2="M/D/YY",  # CA
+        A2EUQ1WTGCTBG2="D/M/YY",  # CA
         A1AM78C64UM0Y8="D/M/YY",  # MX
     )
 
+    NORMALIZED_FIELD_NAMES = ["date", "rating", "comments", "response", "order_id", "rater_email"]
+
     name = "GET_SELLER_FEEDBACK_DATA"
-    cursor_field = "Date"
+    cursor_field = "date"
     transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization)
 
     def __init__(self, *args, **kwargs):
@@ -566,6 +568,19 @@ def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any
 
         return transform_function
 
+    # csv header field names for this report differ per marketplace (are localized to marketplace language)
+    # but columns come in the same order
+    # so we set fieldnames to our custom ones
+    # and raise error if original and custom header field count does not match
+    @staticmethod
+    def parse_document(document):
+        reader = csv.DictReader(StringIO(document), delimiter="\t", fieldnames=SellerFeedbackReports.NORMALIZED_FIELD_NAMES)
+        original_fieldnames = next(reader)
+        if len(original_fieldnames) != len(SellerFeedbackReports.NORMALIZED_FIELD_NAMES):
+            raise ValueError("Original and normalized header field count does not match")
+
+        return reader
+
 
 class Orders(IncrementalAmazonSPStream):
     """

diff --git a/...ntegrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py b/...ntegrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py
@@ -30,6 +30,8 @@ def reports_stream(marketplace_id):
 
 
 INPUT_DATES = {
+    "YYYY-MM-DD": ["2017-01-13", "2017-12-12", "2017-12-17", "2011-12-13"],
+    "D.M.YY": ["13.1.17", "12.12.17", "17.12.17", "13.12.11"],
     "YY/M/D": ["17/1/13", "17/12/12", "17/12/17", "11/12/13"],
     "D/M/YY": ["13/1/17", "12/12/17", "17/12/17", "13/12/11"],
     "M/D/YY": ["1/13/17", "12/12/17", "12/17/17", "12/13/11"],
@@ -45,8 +47,8 @@ def parametrize_seller_feedback():
             result.append(
                 (
                     marketplace_id,
-                    {"Date": input_date, "Rating": 1, "Comments": "c", "Response": "r", "Order ID": "1", "Rater Email": "e"},
-                    {"Date": expected_date, "Rating": 1, "Comments": "c", "Response": "r", "Order ID": "1", "Rater Email": "e"},
+                    {"date": input_date, "rating": 1, "comments": "c", "response": "r", "order_id": "1", "rater_email": "e"},
+                    {"date": expected_date, "rating": 1, "comments": "c", "response": "r", "order_id": "1", "rater_email": "e"},
                 )
             )
 

diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md
@@ -64,6 +64,7 @@ Information about rate limits you may find [here](https://github.com/amzn/sellin
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| `0.2.9` | 2021-12-30 | [\#9212](https://github.com/airbytehq/airbyte/pull/9212) | Normalize GET_SELLER_FEEDBACK_DATA header field names |
 | `0.2.8` | 2021-12-22 | [\#8810](https://github.com/airbytehq/airbyte/pull/8810) | Fix GET_SELLER_FEEDBACK_DATA Date cursor field format |
 | `0.2.7` | 2021-12-21 | [\#9002](https://github.com/airbytehq/airbyte/pull/9002) | Extract REPORTS_MAX_WAIT_SECONDS to configurable parameter |
 | `0.2.6` | 2021-12-10 | [\#8179](https://github.com/airbytehq/airbyte/pull/8179) | Add GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT report |