CI: Enable testing for BigQuery

Author: erezsh

.github/workflows/ci.yml

@@ -42,11 +42,29 @@ jobs:
       - name: Install package
         run: "poetry install"
 
+      # BigQuery start
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+
+      - name: 'Set up BigQuery Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v1'
+
+    #   - name: 'Use gcloud CLI'
+    #     run: "gcloud config configurations list"
+
+      - name: "Install BigQuery for Python"
+        run: poetry add google-cloud-bigquery
+
+      # BigQuery end
+
       - name: Run unit tests
         env:
             DATADIFF_SNOWFLAKE_URI: '${{ secrets.DATADIFF_SNOWFLAKE_URI }}'
             DATADIFF_PRESTO_URI: '${{ secrets.DATADIFF_PRESTO_URI }}'
             DATADIFF_TRINO_URI: '${{ secrets.DATADIFF_TRINO_URI }}'
+            DATADIFF_BIGQUERY_URI: '${{ secrets.DATADIFF_BIGQUERY_URI }}'
             DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse'
             DATADIFF_VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica'
         run: |

.github/workflows/ci_full.yml

@@ -39,12 +39,30 @@ jobs:
       - name: Install package
         run: "poetry install"
 
+      # BigQuery start
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+
+      - name: 'Set up BigQuery Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v1'
+
+    #   - name: 'Use gcloud CLI'
+    #     run: "gcloud config configurations list"
+
+      - name: "Install BigQuery for Python"
+        run: poetry add google-cloud-bigquery
+
+      # BigQuery end
+
       - name: Run unit tests
         env:
             DATADIFF_SNOWFLAKE_URI: '${{ secrets.DATADIFF_SNOWFLAKE_URI }}'
             DATADIFF_PRESTO_URI: '${{ secrets.DATADIFF_PRESTO_URI }}'
             DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse'
             DATADIFF_VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica'
+            DATADIFF_BIGQUERY_URI: '${{ secrets.DATADIFF_BIGQUERY_URI }}'
         run: |
           chmod +x tests/waiting_for_stack_up.sh
           ./tests/waiting_for_stack_up.sh && TEST_ACROSS_ALL_DBS=full poetry run unittest-parallel -j 16

pyproject.toml

@@ -51,6 +51,7 @@ presto-python-client = "*"
 clickhouse-driver = "*"
 vertica-python = "*"
 duckdb = "^0.6.0"
+# google-cloud-bigquery = "*"
 # databricks-sql-connector = "*"
 
 [tool.poetry.extras]

tests/common.py

@@ -24,7 +24,7 @@
 TEST_POSTGRESQL_CONN_STRING: str = "postgresql://postgres:Password1@localhost/postgres"
 TEST_SNOWFLAKE_CONN_STRING: str = os.environ.get("DATADIFF_SNOWFLAKE_URI") or None
 TEST_PRESTO_CONN_STRING: str = os.environ.get("DATADIFF_PRESTO_URI") or None
-TEST_BIGQUERY_CONN_STRING: str = None
+TEST_BIGQUERY_CONN_STRING: str = os.environ.get("DATADIFF_BIGQUERY_URI") or None
 TEST_REDSHIFT_CONN_STRING: str = None
 TEST_ORACLE_CONN_STRING: str = None
 TEST_DATABRICKS_CONN_STRING: str = os.environ.get("DATADIFF_DATABRICKS_URI")
@@ -97,7 +97,7 @@ def _print_used_dbs():
     used = {k.__name__ for k, v in CONN_STRINGS.items() if v is not None}
     unused = {k.__name__ for k, v in CONN_STRINGS.items() if v is None}
 
-    logging.info(f"Testing databases: {', '.join(used)}")
+    print(f"Testing databases: {', '.join(used)}")
     if unused:
         logging.info(f"Connection not configured; skipping tests for: {', '.join(unused)}")
     if TEST_ACROSS_ALL_DBS: