Merge pull request #23 from ingestly/add-new-cols-20210722

Added new columns & Added support for PCM
ingestly · Jul 21, 2021 · f38ea15 · f38ea15
2 parents 03c1f5e + 20470d3
commit f38ea15
Show file tree

Hide file tree

Showing 7 changed files with 58 additions and 15 deletions.
diff --git a/BigQuery/log_format b/BigQuery/log_format
@@ -2,7 +2,7 @@
   "timestamp":"%{begin:%Y-%m-%dT%H:%M:%S}t.%{msec_frac}t",
   "sdk_version":"%{json.escape(urldecode(subfield(req.url.qs, "sdk", "&")))}V",
   "sdk_api_key":"%{json.escape(urldecode(subfield(req.url.qs, "key", "&")))}V",
-  "endpoint_version":"1.0.0",
+  "endpoint_version":"1.1.0",
   "action":"%{json.escape(urldecode(subfield(req.url.qs, "action", "&")))}V",
   "category":"%{json.escape(urldecode(subfield(req.url.qs, "category", "&")))}V",
   "cookie":"%{json.escape(if(subfield(req.url.qs, "ck", "&") != "false", "true", "false"))}V",
@@ -33,9 +33,13 @@
   "screen_height":"%{if(subfield(req.url.qs, "scHeight", "&"), subfield(req.url.qs, "scHeight", "&"), "-1")}V",
   "screen_width":"%{if(subfield(req.url.qs, "scWidth", "&"), subfield(req.url.qs, "scWidth", "&"), "-1")}V",
   "screen_orientation":"%{json.escape(urldecode(subfield(req.url.qs, "scOrientation", "&")))}V",
-  "device_type":"%{json.escape(urldecode(subfield(req.url.qs, "dvType", "&")))}V",
-  "device_os":"%{json.escape(urldecode(subfield(req.url.qs, "dvOs", "&")))}V",
-  "device_platform":"%{json.escape(urldecode(subfield(req.url.qs, "dvPlatform", "&")))}V",
+  "device_type":"%{json.escape(client.platform.hwtype)}V",
+  "device_is_bot":"%{client.class.bot}V",
+  "device_bot_name":"%{json.escape(client.bot.name)}V",
+  "device_os_name":"%{json.escape(client.os.name)}V",
+  "device_os_version":"%{json.escape(client.os.version)}V",
+  "device_browser_name":"%{json.escape(client.browser.name)}V",
+  "device_browser_version":"%{json.escape(client.browser.version)}V",
   "timing_interactive":"%{if(subfield(req.url.qs, "ptTti", "&"), json.escape(subfield(req.url.qs, "ptTti", "&")), "-1")}V",
   "timing_dcl":"%{if(subfield(req.url.qs, "ptDcl", "&"), json.escape(subfield(req.url.qs, "ptDcl", "&")), "-1")}V",
   "timing_complete":"%{if(subfield(req.url.qs, "ptComplete", "&"), json.escape(subfield(req.url.qs, "ptComplete", "&")), "-1")}V",
@@ -106,6 +110,7 @@
   "geo_region":"%{client.geo.region}V",
   "geo_country_name":"%{client.geo.country_name}V",
   "geo_country_code":"%{client.geo.country_code}V",
+  "geo_postal_code":"%{client.geo.postal_code}V",
   "geo_city":"%{client.geo.city}V",
   "geo_latitude":"%{client.geo.latitude}V",
   "geo_longitude":"%{client.geo.longitude}V",

diff --git a/BigQuery/table_schema b/BigQuery/table_schema
@@ -175,12 +175,32 @@
         "mode": "NULLABLE"
     },
     {
-        "name": "device_os",
+        "name": "device_is_bot",
+        "type": "BOOLEAN",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "device_bot_name",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "device_os_name",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "device_os_version",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "device_browser_name",
         "type": "STRING",
         "mode": "NULLABLE"
     },
     {
-        "name": "device_platform",
+        "name": "device_browser_version",
         "type": "STRING",
         "mode": "NULLABLE"
     },
@@ -533,6 +553,11 @@
         "name": "geo_country_code",
         "type": "STRING",
         "mode": "NULLABLE"
+    },
+        {
+        "name": "geo_postal_code",
+        "type": "STRING",
+        "mode": "NULLABLE"
     },
     {
         "name": "geo_city",

diff --git a/Elasticsearch/log_format b/Elasticsearch/log_format
@@ -2,7 +2,7 @@
   "timestamp":"%{begin:%Y-%m-%dT%H:%M:%S}t.%{msec_frac}t",
   "sdk_version":"%{json.escape(urldecode(subfield(req.url.qs, "sdk", "&")))}V",
   "sdk_api_key":"%{json.escape(urldecode(subfield(req.url.qs, "key", "&")))}V",
-  "endpoint_version":"1.0.0",
+  "endpoint_version":"1.1.0",
   "action":"%{json.escape(urldecode(subfield(req.url.qs, "action", "&")))}V",
   "category":"%{json.escape(urldecode(subfield(req.url.qs, "category", "&")))}V",
   "cookie":"%{json.escape(if(subfield(req.url.qs, "ck", "&") != "false", "true", "false"))}V",
@@ -36,9 +36,13 @@
   },
   "device": {
     "user_agent":"%{json.escape(req.http.User-Agent)}V",
-    "type":"%{json.escape(urldecode(subfield(req.url.qs, "dvType", "&")))}V",
-    "os":"%{json.escape(urldecode(subfield(req.url.qs, "dvOs", "&")))}V",
-    "platform":"%{json.escape(urldecode(subfield(req.url.qs, "dvPlatform", "&")))}V",
+    "type":"%{json.escape(client.platform.hwtype)}V",
+    "is_bot":%{if(client.class.bot, "true", "false")}V,
+    "bot_name":"%{json.escape(client.bot.name)}V",
+    "os_name":"%{json.escape(client.os.name)}V",
+    "os_version":"%{json.escape(client.os.version)}V",
+    "browser_name":"%{json.escape(client.browser.name)}V",
+    "browser_version":"%{json.escape(client.browser.version)}V",
     "viewport_height":"%{if(subfield(req.url.qs, "vpHeight", "&"), subfield(req.url.qs, "vpHeight", "&"), "-1")}V",
     "viewport_width":"%{if(subfield(req.url.qs, "vpWidth", "&"), subfield(req.url.qs, "vpWidth", "&"), "-1")}V",
     "screen_height":"%{if(subfield(req.url.qs, "scHeight", "&"), subfield(req.url.qs, "scHeight", "&"), "-1")}V",
@@ -134,6 +138,7 @@
     "region":"%{client.geo.region}V",
     "country_name":"%{client.geo.country_name}V",
     "country_code":"%{client.geo.country_code}V",
+    "postal_code":"%{client.geo.postal_code}V",
     "city":"%{client.geo.city}V",
     "latitude":"%{client.geo.latitude}V",
     "longitude":"%{client.geo.longitude}V",

diff --git a/Elasticsearch/mapping_template.json b/Elasticsearch/mapping_template.json
@@ -89,6 +89,9 @@
           },
           "viewport_width": {
             "type": "integer"
+          },
+          "is_bot": {
+            "type": "boolean"
           }
         }
       },

diff --git a/S3/athena_create_table.sql b/S3/athena_create_table.sql
@@ -34,8 +34,12 @@ CREATE EXTERNAL TABLE IF NOT EXISTS mydatabase.ingestly (
   `screen_width` double,
   `screen_orientation` string,
   `device_type` string,
-  `device_os` string,
-  `device_platform` string,
+  `device_is_bot` string,
+  `device_bot_name` string,
+  `device_os_name` string,
+  `device_os_version` string,
+  `device_browser_name` string,
+  `device_browser_version` string,
   `timing_interactive` bigint,
   `timing_dcl` bigint,
   `timing_complete` bigint,
@@ -106,6 +110,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS mydatabase.ingestly (
   `geo_region` string,
   `geo_country_name` string,
   `geo_country_code` string,
+  `geo_postal_code` string,
   `geo_city` string,
   `geo_latitude` double,
   `geo_longitude` double,

diff --git a/S3/log_format b/S3/log_format
diff --git a/ingestly.vcl b/ingestly.vcl
@@ -14,7 +14,7 @@ sub vcl_recv {
     if(req.url ~ "^/\.well-known/(attribution-reporting|private-click-measurement)/.*"){
       # Attribution Reporting & Private Click Measurement (Report Endpoint)
       error 200 "OK";
-    }elseif(req.url ~ "^/ingestly-attribution/pcm/\?.*"){
+    }elseif(req.url ~ "^/ingestly-pixel/.*?/\?.*"){
       # Private Click Measurement (Redirector)
       error 302 "Found";
     }else{
@@ -119,7 +119,7 @@ sub vcl_error {
     }
     return (deliver);
 
-  }elseif(req.url ~ "^/ingestly-attribution/pcm/\?.*"){
+  }elseif(req.url ~ "^/ingestly-pixel/pcm/\?.*"){
     # Private Click Measurement (Redirector)
     set obj.http.Location = "https://" + req.http.host + "/.well-known/private-click-measurement/trigger-attribution/" + subfield(req.url.qs, "trigger_data", "&") + "/" + subfield(req.url.qs, "trigger_priority", "&");
     return (deliver);