Simplify running the integration tests

.circleci/config.yml

@@ -18,69 +18,29 @@ jobs:
       - restore_cache:
           key: deps1-{{ .Branch }}
 
-      - run:
-          name: "Setup dbt"
-          command: |
-            python3 -m venv venv
-            . venv/bin/activate
-
-            pip install --upgrade pip setuptools
-            pip install dbt
-
-            mkdir -p ~/.dbt
-            cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
-
       - run:
           name: "Run Tests - Postgres"
           environment:
+            CI_DBT_HOST: localhost
             CI_DBT_USER: root
             CI_DBT_PASS: ''
             CI_DBT_PORT: 5432
             CI_DBT_DBNAME: circle_test
-          command: |
-            . venv/bin/activate
-            cd integration_tests
-            dbt deps --target postgres
-            dbt seed --target postgres --full-refresh
-            dbt run --target postgres
-            dbt test --target postgres
+          command: ./run_test.sh postgres
 
       - run:
           name: "Run Tests - Redshift"
-          command: |
-            . venv/bin/activate
-            echo `pwd`
-            cd integration_tests
-            dbt deps --target redshift
-            dbt seed --target redshift --full-refresh
-            dbt run --target redshift
-            dbt test --target redshift
+          command: ./run_test.sh redshift
 
       - run:
           name: "Run Tests - Snowflake"
-          command: |
-            . venv/bin/activate
-            echo `pwd`
-            cd integration_tests
-            dbt deps --target snowflake
-            dbt seed --target snowflake --full-refresh
-            dbt run --target snowflake
-            dbt test --target snowflake
+          command: ./run_test.sh snowflake
 
       - run:
           name: "Run Tests - BigQuery"
           environment:
               GCLOUD_SERVICE_KEY_PATH: "/home/circleci/gcloud-service-key.json"
-
-          command: |
-            . venv/bin/activate
-            echo `pwd`
-            cd integration_tests
-            dbt deps --target bigquery
-            dbt seed --target bigquery --full-refresh
-            dbt run --target bigquery
-            dbt test --target bigquery
-
+          command: ./run_test.sh bigquery
 
       - save_cache:
           key: deps1-{{ .Branch }}

.gitignore

@@ -2,3 +2,4 @@
 target/
 dbt_modules/
 logs/
+venv/

docker-compose.yml

@@ -0,0 +1,32 @@
+version: "3.7"
+services:
+
+  dbt:
+    image: circleci/python:3.6.2-stretch
+    depends_on:
+      - ${TARGET}
+    env_file: "./integration_tests/.env/${TARGET}.env"
+    entrypoint: "/repo/run_test.sh ${TARGET}"
+    working_dir: /repo
+    volumes:
+        - ".:/repo"
+
+  postgres:
+    image: circleci/postgres:9.6.5-alpine-ram
+    ports:
+      - "5432:5432"
+
+  # dummy container, since snowflake is a managed service
+  snowflake:
+    image: circleci/python:3.6.2-stretch
+    entrypoint: "/bin/true"
+
+  # dummy container, since bigquery is a managed service
+  bigquery:
+    image: circleci/python:3.6.2-stretch
+    entrypoint: "/bin/true"
+
+  # dummy container, since redshift is a managed service
+  redshift:
+    image: circleci/python:3.6.2-stretch
+    entrypoint: "/bin/true"

integration_tests/.env/bigquery.env

@@ -0,0 +1 @@
+GCLOUD_SERVICE_KEY_PATH=

integration_tests/.env/postgres.env

@@ -0,0 +1,5 @@
+CI_DBT_HOST=postgres
+CI_DBT_USER=root
+CI_DBT_PASS=''
+CI_DBT_PORT=5432
+CI_DBT_DBNAME=circle_test

integration_tests/.env/redshift.env

@@ -0,0 +1,4 @@
+CI_REDSHIFT_DBT_HOST=
+CI_REDSHIFT_DBT_USER=
+CI_REDSHIFT_DBT_PASS=
+CI_REDSHIFT_DBT_DBNAME=

integration_tests/.env/snowflake.env

@@ -0,0 +1,6 @@
+CI_SNOWFLAKE_DBT_ACCOUNT=
+CI_SNOWFLAKE_DBT_USER=
+CI_SNOWFLAKE_DBT_PASS=
+CI_SNOWFLAKE_DBT_ROLE=
+CI_SNOWFLAKE_DBT_DATABASE=
+CI_SNOWFLAKE_DBT_WAREHOUSE=

integration_tests/Makefile

@@ -1,23 +1,12 @@
+TARGETS = postgres redshift snowflake bigquery
+.PHONY : test test-all $(TARGETS)
 
-test-postgres:
-	dbt seed --target postgres --full-refresh
-	dbt run --target postgres --full-refresh --exclude test_insert_by_period
-	dbt test --target postgres --exclude test_insert_by_period
+test: export TARGET = $(target)
+test:
+	docker-compose -f ../docker-compose.yml up dbt
 
-test-redshift:
-	dbt seed --target redshift --full-refresh
-	dbt run --target redshift --full-refresh
-	dbt test --target redshift
+$(TARGETS):
+	$(MAKE) test target=$@
 
-test-snowflake:
-	dbt seed --target snowflake --full-refresh
-	dbt run --target snowflake --full-refresh
-	dbt test --target snowflake
-
-test-bigquery:
-	dbt seed --target bigquery --full-refresh
-	dbt run --target bigquery --full-refresh
-	dbt test --target bigquery
-
-test-all: test-postgres test-redshift test-snowflake test-bigquery
-	echo "Completed successfully"
+test-all: $(TARGETS)
+	echo "Tested all targets"

integration_tests/README.md

@@ -1,4 +1,22 @@
-### dbt integration test suite for dbt-utils
+###  Run the integration tests
+
+To run the integration tests on your local machine, like they will get run in the CI (using CircleCI), you can do the following:
+
+Assuming you are in the `integration_tests` folder:
+
+```bash
+make test target=[postgres|redshift|...]
+```
+
+or, to test against all targets:
+
+```bash
+make test-all
+```
+
+Where possible, targets are being run in docker containers (this works for Postgres or in the future Spark for example). For managed services like Snowflake, BigQuery and Redshift this is not possible, hence your own configuration for these services has to be provided in the appropriate env files in `integration_tests/.env/[TARGET].env`
+
+### Creating a new integration test
 
 This directory contains an example dbt project which tests the macros in the `dbt-utils` package. An integration test typically involves making 1) a new seed file 2) a new model file 3) a schema test.
 
@@ -18,4 +36,4 @@ $ dbt run --model {your_model_name}
 $ dbt test --model {your_model_name}
 ```
 
-If the tests all pass, then you're good to go! All tests will be run automatically when you create a PR against this repo.
+If the tests all pass, then you're good to go! All tests will be run automatically when you create a PR against this repo.

integration_tests/ci/sample.profiles.yml

@@ -11,7 +11,7 @@ integration_tests:
   outputs:
     postgres:
       type: postgres
-      host: localhost
+      host: "{{ env_var('CI_DBT_HOST') }}"
       user: "{{ env_var('CI_DBT_USER') }}"
       pass: "{{ env_var('CI_DBT_PASS') }}"
       port: "{{ env_var('CI_DBT_PORT') }}"
@@ -46,4 +46,4 @@ integration_tests:
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
       schema: dbt_utils_integration_tests_snowflake
-      threads: 1
+      threads: 1

run_test.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+VENV="venv/bin/activate"
+
+if [[ ! -f $VENV ]]; then
+    python3 -m venv venv
+    . $VENV
+
+    pip install --upgrade pip setuptools
+    pip install dbt
+fi
+
+. $VENV
+cd integration_tests
+
+mkdir -p ~/.dbt
+cp ci/sample.profiles.yml ~/.dbt/profiles.yml
+
+dbt deps --target $1
+dbt seed --target $1 --full-refresh
+dbt run --target $1
+dbt test --target $1