From eff7a69aff68b686098ee9612f1c58036eec1e04 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 3 Aug 2020 19:37:44 -0400 Subject: [PATCH 01/18] Replace adapter_macro with adapter.dispatch --- dbt_project.yml | 2 +- integration_tests/Makefile | 34 ++++++++++++------ .../cross_db_utils/_get_utils_namespaces.sql | 4 +++ macros/cross_db_utils/concat.sql | 10 +++--- macros/cross_db_utils/current_timestamp.sql | 4 +-- macros/cross_db_utils/datatypes.sql | 12 +++---- macros/cross_db_utils/date_trunc.sql | 2 +- macros/cross_db_utils/dateadd.sql | 9 +---- macros/cross_db_utils/datediff.sql | 35 +------------------ macros/cross_db_utils/except.sql | 2 +- macros/cross_db_utils/hash.sql | 2 +- macros/cross_db_utils/identifier.sql | 2 +- macros/cross_db_utils/intersect.sql | 2 +- macros/cross_db_utils/last_day.sql | 2 +- macros/cross_db_utils/length.sql | 2 +- macros/cross_db_utils/literal.sql | 2 +- macros/cross_db_utils/position.sql | 2 +- macros/cross_db_utils/replace.sql | 2 +- macros/cross_db_utils/right.sql | 2 +- macros/cross_db_utils/safe_cast.sql | 2 +- macros/cross_db_utils/split_part.sql | 2 +- macros/cross_db_utils/width_bucket.sql | 2 +- macros/sql/get_tables_by_prefix_sql.sql | 2 +- 23 files changed, 58 insertions(+), 82 deletions(-) create mode 100644 macros/cross_db_utils/_get_utils_namespaces.sql diff --git a/dbt_project.yml b/dbt_project.yml index 03a57409..e9a847d3 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ name: 'dbt_utils' version: '0.1.0' -require-dbt-version: [">=0.17.0", "<0.18.0"] +require-dbt-version: [">=0.18.0", "<0.19.0"] config-version: 2 target-path: "target" diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 89cc0fbc..325ed1d9 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -1,14 +1,26 @@ -TARGETS = postgres redshift snowflake bigquery -.PHONY : test test-all $(TARGETS) +test-postgres: + dbt seed --target postgres --full-refresh + dbt run --target postgres --full-refresh --vars 'update: false' + dbt run --target postgres --vars 'update: true' + dbt test --target postgres -test: export TARGET = $(target) -test: export MODELS = $(models) -test: export SEEDS = $(seeds) -test: - docker-compose -f ../docker-compose.yml up dbt +test-redshift: + dbt seed --target redshift --full-refresh + dbt run --target redshift --full-refresh --vars 'update: false' + dbt run --target redshift --vars 'update: true' + dbt test --target redshift -$(TARGETS): - $(MAKE) test target=$@ +test-snowflake: + dbt seed --target snowflake --full-refresh + dbt run --target snowflake --full-refresh --vars 'update: false' + dbt run --target snowflake --vars 'update: true' + dbt test --target snowflake -test-all: $(TARGETS) - echo "Tested all targets" +test-bigquery: + dbt seed --target bigquery --full-refresh + dbt run --target bigquery --full-refresh --vars 'update: false' + dbt run --target bigquery --vars 'update: true' + dbt test --target bigquery + +test-all: test-postgres test-redshift test-snowflake test-bigquery + echo "Completed successfully" diff --git a/macros/cross_db_utils/_get_utils_namespaces.sql b/macros/cross_db_utils/_get_utils_namespaces.sql new file mode 100644 index 00000000..4296f86e --- /dev/null +++ b/macros/cross_db_utils/_get_utils_namespaces.sql @@ -0,0 +1,4 @@ +{% macro _get_utils_namespaces() %} + {% set override_namespaces = var('utils_namespaces', []) %} + {% do return(override_namespaces + ['dbt_utils']) %} +{% endmacro %} diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 88512611..07bc6483 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,23 +1,23 @@ + + {% macro concat(fields) -%} - {{ adapter_macro('dbt_utils.concat', fields) }} + {{ adapter.dispatch('concat', packages = dbt_utils._get_utils_namespaces())(fields) }} {%- endmacro %} - {% macro default__concat(fields) -%} concat({{ fields|join(', ') }}) {%- endmacro %} - {% macro alternative_concat(fields) %} {{ fields|join(' || ') }} {% endmacro %} {% macro redshift__concat(fields) %} - {{dbt_utils.alternative_concat(fields)}} + {{ alternative_concat(fields) }} {% endmacro %} {% macro snowflake__concat(fields) %} - {{dbt_utils.alternative_concat(fields)}} + {{ alternative_concat(fields) }} {% endmacro %} diff --git a/macros/cross_db_utils/current_timestamp.sql b/macros/cross_db_utils/current_timestamp.sql index d838dce9..090e246d 100644 --- a/macros/cross_db_utils/current_timestamp.sql +++ b/macros/cross_db_utils/current_timestamp.sql @@ -1,5 +1,5 @@ {% macro current_timestamp() -%} - {{ adapter_macro('dbt_utils.current_timestamp') }} + {{ adapter.dispatch('current_timestamp', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro %} {% macro default__current_timestamp() %} @@ -17,7 +17,7 @@ {% macro current_timestamp_in_utc() -%} - {{ adapter_macro('dbt_utils.current_timestamp_in_utc') }} + {{ adapter.dispatch('current_timestamp_in_utc', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro %} {% macro default__current_timestamp_in_utc() %} diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index 24632ddd..1f949281 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -1,7 +1,7 @@ {# string ------------------------------------------------- #} {%- macro type_string() -%} - {{ adapter_macro('dbt_utils.type_string') }} + {{ adapter.dispatch('type_string', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_string() %} @@ -25,7 +25,7 @@ {# timestamp ------------------------------------------------- #} {%- macro type_timestamp() -%} - {{ adapter_macro('dbt_utils.type_timestamp') }} + {{ adapter.dispatch('type_timestamp', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_timestamp() %} @@ -40,7 +40,7 @@ {# float ------------------------------------------------- #} {%- macro type_float() -%} - {{ adapter_macro('dbt_utils.type_float') }} + {{ adapter.dispatch('type_float', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_float() %} @@ -54,7 +54,7 @@ {# numeric ------------------------------------------------ #} {%- macro type_numeric() -%} - {{ adapter_macro('dbt_utils.type_numeric') }} + {{ adapter.dispatch('type_numeric', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_numeric() %} @@ -69,7 +69,7 @@ {# bigint ------------------------------------------------- #} {%- macro type_bigint() -%} - {{ adapter_macro('dbt_utils.type_bigint') }} + {{ adapter.dispatch('type_bigint', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_bigint() %} @@ -83,7 +83,7 @@ {# int ------------------------------------------------- #} {%- macro type_int() -%} - {{ adapter_macro('dbt_utils.type_int') }} + {{ adapter.dispatch('type_int', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_int() %} diff --git a/macros/cross_db_utils/date_trunc.sql b/macros/cross_db_utils/date_trunc.sql index d6073739..08653134 100644 --- a/macros/cross_db_utils/date_trunc.sql +++ b/macros/cross_db_utils/date_trunc.sql @@ -1,5 +1,5 @@ {% macro date_trunc(datepart, date) -%} - {{ adapter_macro('dbt_utils.date_trunc', datepart, date) }} + {{ adapter.dispatch('date_trunc', packages = dbt_utils._get_utils_namespaces()) (datepart, date) }} {%- endmacro %} {% macro default__date_trunc(datepart, date) %} diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 92977df8..8df7fc77 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -1,5 +1,5 @@ {% macro dateadd(datepart, interval, from_date_or_timestamp) %} - {{ adapter_macro('dbt_utils.dateadd', datepart, interval, from_date_or_timestamp) }} + {{ adapter.dispatch('dateadd', packages = dbt_utils._get_utils_namespaces())(datepart, interval, from_date_or_timestamp) }} {% endmacro %} @@ -22,10 +22,3 @@ ) {% endmacro %} - - -{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %} - - {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) - -{% endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index c0c1e7d4..44c6052f 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -1,5 +1,5 @@ {% macro datediff(first_date, second_date, datepart) %} - {{ adapter_macro('dbt_utils.datediff', first_date, second_date, datepart) }} + {{ adapter.dispatch('datediff', packages = dbt_utils._get_utils_namespaces())(first_date, second_date, datepart) }} {% endmacro %} @@ -24,36 +24,3 @@ {% endmacro %} - -{% macro postgres__datediff(first_date, second_date, datepart) %} - - {% if datepart == 'year' %} - (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date)) - {% elif datepart == 'quarter' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) - {% elif datepart == 'month' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date)) - {% elif datepart == 'day' %} - (({{second_date}})::date - ({{first_date}})::date) - {% elif datepart == 'week' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} / 7 + case - when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then - case when {{first_date}} <= {{second_date}} then 0 else -1 end - else - case when {{first_date}} <= {{second_date}} then 1 else 0 end - end) - {% elif datepart == 'hour' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp)) - {% elif datepart == 'minute' %} - ({{ dbt_utils.datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp)) - {% elif datepart == 'second' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp))) - {% elif datepart == 'millisecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp))) - {% elif datepart == 'microsecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp))) - {% else %} - {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }} - {% endif %} - -{% endmacro %} diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql index 95a022f2..c9e8206d 100644 --- a/macros/cross_db_utils/except.sql +++ b/macros/cross_db_utils/except.sql @@ -1,5 +1,5 @@ {% macro except() %} - {{ adapter_macro('dbt_utils.except') }} + {{ adapter.dispatch('except', packages = dbt_utils._get_utils_namespaces())() }} {% endmacro %} diff --git a/macros/cross_db_utils/hash.sql b/macros/cross_db_utils/hash.sql index b7d2c235..8244b7f8 100644 --- a/macros/cross_db_utils/hash.sql +++ b/macros/cross_db_utils/hash.sql @@ -1,5 +1,5 @@ {% macro hash(field) -%} - {{ adapter_macro('dbt_utils.hash', field) }} + {{ adapter.dispatch('hash', packages = dbt_utils._get_utils_namespaces()) (field) }} {%- endmacro %} diff --git a/macros/cross_db_utils/identifier.sql b/macros/cross_db_utils/identifier.sql index ede09f66..1a379261 100644 --- a/macros/cross_db_utils/identifier.sql +++ b/macros/cross_db_utils/identifier.sql @@ -4,7 +4,7 @@ Use `adapter.quote` instead. The {}.{} model triggered this warning. \ '.format(model.package_name, model.name) -%} {%- do exceptions.warn(error_message) -%} - {{ adapter_macro('dbt_utils.identifier', value) }} + {{ adapter.dispatch('identifier', packages = dbt_utils._get_utils_namespaces()) (value) }} {% endmacro %} {% macro default__identifier(value) -%} diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql index c2853bd0..252479a0 100644 --- a/macros/cross_db_utils/intersect.sql +++ b/macros/cross_db_utils/intersect.sql @@ -1,5 +1,5 @@ {% macro intersect() %} - {{ adapter_macro('dbt_utils.intersect') }} + {{ adapter.dispatch('intersect', packages = dbt_utils._get_utils_namespaces())() }} {% endmacro %} diff --git a/macros/cross_db_utils/last_day.sql b/macros/cross_db_utils/last_day.sql index 0efe6257..1a58b51b 100644 --- a/macros/cross_db_utils/last_day.sql +++ b/macros/cross_db_utils/last_day.sql @@ -4,7 +4,7 @@ testing is required to validate that it will work on other dateparts. */ {% macro last_day(date, datepart) %} - {{ adapter_macro('dbt_utils.last_day', date, datepart) }} + {{ adapter.dispatch('last_day', packages = dbt_utils._get_utils_namespaces()) (date, datepart) }} {% endmacro %} diff --git a/macros/cross_db_utils/length.sql b/macros/cross_db_utils/length.sql index a2bfc8f0..a2595d2e 100644 --- a/macros/cross_db_utils/length.sql +++ b/macros/cross_db_utils/length.sql @@ -1,5 +1,5 @@ {% macro length(expression) -%} - {{ adapter_macro('dbt_utils.length', expression) }} + {{ adapter.dispatch('length', packages = dbt_utils._get_utils_namespaces()) (expression) }} {% endmacro %} diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql index 2d29b36f..b486fc2f 100644 --- a/macros/cross_db_utils/literal.sql +++ b/macros/cross_db_utils/literal.sql @@ -1,6 +1,6 @@ {%- macro string_literal(value) -%} - {{ adapter_macro('dbt_utils.string_literal', value) }} + {{ adapter.dispatch('string_literal', packages = dbt_utils._get_utils_namespaces()) (value) }} {%- endmacro -%} {% macro default__string_literal(value) -%} diff --git a/macros/cross_db_utils/position.sql b/macros/cross_db_utils/position.sql index df4a58bb..a5b70c3a 100644 --- a/macros/cross_db_utils/position.sql +++ b/macros/cross_db_utils/position.sql @@ -1,5 +1,5 @@ {% macro position(substring_text, string_text) -%} - {{ adapter_macro('dbt_utils.position', substring_text, string_text) }} + {{ adapter.dispatch('position', packages = dbt_utils._get_utils_namespaces()) (substring_text, string_text) }} {% endmacro %} diff --git a/macros/cross_db_utils/replace.sql b/macros/cross_db_utils/replace.sql index 00b2837d..f1dc363d 100644 --- a/macros/cross_db_utils/replace.sql +++ b/macros/cross_db_utils/replace.sql @@ -1,5 +1,5 @@ {% macro replace(field, old_chars, new_chars) -%} - {{ adapter_macro('dbt_utils.replace', field, old_chars, new_chars) }} + {{ adapter.dispatch('replace', packages = dbt_utils._get_utils_namespaces()) (field, old_chars, new_chars) }} {% endmacro %} diff --git a/macros/cross_db_utils/right.sql b/macros/cross_db_utils/right.sql index 5a2d5226..992b848b 100644 --- a/macros/cross_db_utils/right.sql +++ b/macros/cross_db_utils/right.sql @@ -1,5 +1,5 @@ {% macro right(string_text, length_expression) -%} - {{ adapter_macro('dbt_utils.right', string_text, length_expression) }} + {{ adapter.dispatch('right', packages = dbt_utils._get_utils_namespaces()) (string_text, length_expression) }} {% endmacro %} {% macro default__right(string_text, length_expression) %} diff --git a/macros/cross_db_utils/safe_cast.sql b/macros/cross_db_utils/safe_cast.sql index 592a0d39..1c3ef532 100644 --- a/macros/cross_db_utils/safe_cast.sql +++ b/macros/cross_db_utils/safe_cast.sql @@ -1,5 +1,5 @@ {% macro safe_cast(field, type) %} - {{ adapter_macro('dbt_utils.safe_cast', field, type) }} + {{ adapter.dispatch('safe_cast', packages = dbt_utils._get_utils_namespaces()) (field, type) }} {% endmacro %} diff --git a/macros/cross_db_utils/split_part.sql b/macros/cross_db_utils/split_part.sql index 6434cf91..0f1b2f22 100644 --- a/macros/cross_db_utils/split_part.sql +++ b/macros/cross_db_utils/split_part.sql @@ -1,5 +1,5 @@ {% macro split_part(string_text, delimiter_text, part_number) %} - {{ adapter_macro('dbt_utils.split_part', string_text, delimiter_text, part_number) }} + {{ adapter.dispatch('split_part', packages = dbt_utils._get_utils_namespaces()) (string_text, delimiter_text, part_number) }} {% endmacro %} diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index 39e0b0b6..8e714410 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -1,5 +1,5 @@ {% macro width_bucket(expr, min_value, max_value, num_buckets) %} - {{ adapter_macro('dbt_utils.width_bucket', expr, min_value, max_value, num_buckets) }} + {{ adapter.dispatch('width_bucket', packages = dbt_utils._get_utils_namespaces()) (expr, min_value, max_value, num_buckets) }} {% endmacro %} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index 2f70bf95..e054012a 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -1,5 +1,5 @@ {% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - {{ adapter_macro('dbt_utils.get_tables_by_prefix_sql', schema, prefix, exclude, database) }} + {{ adapter.dispatch('get_tables_by_prefix_sql', packages = dbt_utils._get_utils_namespaces()) (schema, prefix, exclude, database) }} {% endmacro %} {% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} From 0e8e70386821d0d50a0d85e5d4e9ceb370e72d30 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 14 Aug 2020 19:54:58 -0400 Subject: [PATCH 02/18] Refactor to get tests passing --- .../models/datetime/test_date_spine.sql | 3 +- .../sql/test_get_relations_by_pattern.sql | 2 +- macros/cross_db_utils/concat.sql | 4 +-- macros/sql/get_relations_by_pattern.sql | 16 ++++++++-- macros/sql/get_tables_by_pattern.sql | 12 ------- macros/sql/get_tables_by_pattern_sql.sql | 28 +++++++++++++++++ macros/sql/get_tables_by_prefix_sql.sql | 31 +++++-------------- 7 files changed, 53 insertions(+), 43 deletions(-) delete mode 100644 macros/sql/get_tables_by_pattern.sql create mode 100644 macros/sql/get_tables_by_pattern_sql.sql diff --git a/integration_tests/models/datetime/test_date_spine.sql b/integration_tests/models/datetime/test_date_spine.sql index 4d0d4004..93cd07f1 100644 --- a/integration_tests/models/datetime/test_date_spine.sql +++ b/integration_tests/models/datetime/test_date_spine.sql @@ -9,8 +9,7 @@ with date_spine as ( {% if target.type == 'postgres' %} - {{ log("WARNING: Not testing - datediff macro is unsupported on Postgres", info=True) }} - select * from {{ ref('data_date_spine') }} + {{ dbt_utils.date_spine("day", "'2018-01-01'::date", "'2018-01-10'::date") }} {% elif target.type == 'bigquery' %} select cast(date_day as date) as date_day diff --git a/integration_tests/models/sql/test_get_relations_by_pattern.sql b/integration_tests/models/sql/test_get_relations_by_pattern.sql index 5a51dd05..b10d8eef 100644 --- a/integration_tests/models/sql/test_get_relations_by_pattern.sql +++ b/integration_tests/models/sql/test_get_relations_by_pattern.sql @@ -1,4 +1,4 @@ {{ config(materialized = 'table') }} -{% set relations = dbt_utils.get_relations_by_pattern(target.schema, 'data_events_') %} +{% set relations = dbt_utils.get_relations_by_pattern(target.schema, 'data_events_%') %} {{ dbt_utils.union_relations(relations) }} \ No newline at end of file diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 07bc6483..58435016 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -14,10 +14,10 @@ {% macro redshift__concat(fields) %} - {{ alternative_concat(fields) }} + {{ dbt_utils.alternative_concat(fields) }} {% endmacro %} {% macro snowflake__concat(fields) %} - {{ alternative_concat(fields) }} + {{ dbt_utils.alternative_concat(fields) }} {% endmacro %} diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index 78a766ad..e09a6aba 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -2,7 +2,7 @@ {%- call statement('get_tables', fetch_result=True) %} - {{ dbt_utils.get_tables_by_pattern(schema_pattern, table_pattern, exclude, database) }} + {{ dbt_utils.get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude, database) }} {%- endcall -%} @@ -20,4 +20,16 @@ {{ return([]) }} {%- endif -%} -{% endmacro %} \ No newline at end of file +{% endmacro %} + +{% macro get_tables_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} + {%- set error_message = ' + Warning: the `get_tables_by_pattern` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ + Use the `get_relations_by_prefix` macro instead. \ + The {}.{} model triggered this warning. \ + '.format(model.package_name, model.name) -%} + {%- do exceptions.warn(error_message) -%} + + {{ return(dbt_utils.get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database)) }} + +{% endmacro %} diff --git a/macros/sql/get_tables_by_pattern.sql b/macros/sql/get_tables_by_pattern.sql deleted file mode 100644 index ad388ca4..00000000 --- a/macros/sql/get_tables_by_pattern.sql +++ /dev/null @@ -1,12 +0,0 @@ -{% macro get_tables_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} - - select distinct - table_schema as "table_schema", table_name as "table_name" - from {{database}}.information_schema.tables - where table_schema ilike '{{ schema_pattern }}' - and table_name ilike '{{ table_pattern }}' - and table_name not ilike '{{ exclude }}' - -{% endmacro %} - - diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql new file mode 100644 index 00000000..df47aa23 --- /dev/null +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -0,0 +1,28 @@ +{% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + {{ adapter.dispatch('get_tables_by_pattern_sql', packages = dbt_utils._get_utils_namespaces()) + (schema_pattern, table_pattern, exclude='', database=target.database) }} +{% endmacro %} + +{% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + + select distinct + table_schema as "table_schema", table_name as "table_name" + from {{database}}.information_schema.tables + where table_schema ilike '{{ schema_pattern }}' + and table_name ilike '{{ table_pattern }}' + and table_name not ilike '{{ exclude }}' + +{% endmacro %} + + +{% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + + select distinct + table_schema, table_name + + from {{adapter.quote(database)}}.{{schema}}.INFORMATION_SCHEMA.TABLES + where table_schema = '{{schema_pattern}}' + and lower(table_name) like lower ('{{table_pattern}}') + and lower(table_name) not like lower ('{{exclude}}') + +{% endmacro %} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index e054012a..46e52221 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -1,27 +1,10 @@ {% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - {{ adapter.dispatch('get_tables_by_prefix_sql', packages = dbt_utils._get_utils_namespaces()) (schema, prefix, exclude, database) }} -{% endmacro %} - -{% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - - select distinct - table_schema as "table_schema", table_name as "table_name" - from {{database}}.information_schema.tables - where table_schema ilike '{{ schema }}' - and table_name ilike '{{ prefix }}%' - and table_name not ilike '{{ exclude }}' - -{% endmacro %} - - -{% macro bigquery__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - select distinct - table_schema, table_name - - from {{adapter.quote(database)}}.{{schema}}.INFORMATION_SCHEMA.TABLES - where table_schema = '{{schema}}' - and lower(table_name) like lower ('{{prefix}}%') - and lower(table_name) not like lower ('{{exclude}}') - + {{ dbt_utils.get_tables_by_pattern_sql( + schema_pattern = schema, + table_pattern = prefix ~ '%', + exclude = exclude, + database = database + ) }} + {% endmacro %} From 4dd1bc3bbfca406336d82a046bd6bbf82e4e4992 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 14 Aug 2020 19:55:57 -0400 Subject: [PATCH 03/18] Prefigure rc1 --- run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_test.sh b/run_test.sh index 737e571a..6bede664 100755 --- a/run_test.sh +++ b/run_test.sh @@ -6,7 +6,7 @@ if [[ ! -f $VENV ]]; then . $VENV pip install --upgrade pip setuptools - pip install "dbt>=0.17.0,<0.18.0" + pip install "dbt==0.18.0rc1" fi . $VENV From 9e9407b9de7c48fa320098580a2aac7aa16551b9 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 14 Aug 2020 20:03:00 -0400 Subject: [PATCH 04/18] Rename namespace var --- macros/cross_db_utils/_get_utils_namespaces.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/cross_db_utils/_get_utils_namespaces.sql b/macros/cross_db_utils/_get_utils_namespaces.sql index 4296f86e..e841381c 100644 --- a/macros/cross_db_utils/_get_utils_namespaces.sql +++ b/macros/cross_db_utils/_get_utils_namespaces.sql @@ -1,4 +1,4 @@ {% macro _get_utils_namespaces() %} - {% set override_namespaces = var('utils_namespaces', []) %} + {% set override_namespaces = var('dbt_utils_dispatch_list', []) %} {% do return(override_namespaces + ['dbt_utils']) %} {% endmacro %} From 25eba99bfc84233d511493220178dd10628611d9 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 17 Aug 2020 10:01:20 -0400 Subject: [PATCH 05/18] Revert makefile changes --- integration_tests/Makefile | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 325ed1d9..89cc0fbc 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -1,26 +1,14 @@ -test-postgres: - dbt seed --target postgres --full-refresh - dbt run --target postgres --full-refresh --vars 'update: false' - dbt run --target postgres --vars 'update: true' - dbt test --target postgres +TARGETS = postgres redshift snowflake bigquery +.PHONY : test test-all $(TARGETS) -test-redshift: - dbt seed --target redshift --full-refresh - dbt run --target redshift --full-refresh --vars 'update: false' - dbt run --target redshift --vars 'update: true' - dbt test --target redshift +test: export TARGET = $(target) +test: export MODELS = $(models) +test: export SEEDS = $(seeds) +test: + docker-compose -f ../docker-compose.yml up dbt -test-snowflake: - dbt seed --target snowflake --full-refresh - dbt run --target snowflake --full-refresh --vars 'update: false' - dbt run --target snowflake --vars 'update: true' - dbt test --target snowflake +$(TARGETS): + $(MAKE) test target=$@ -test-bigquery: - dbt seed --target bigquery --full-refresh - dbt run --target bigquery --full-refresh --vars 'update: false' - dbt run --target bigquery --vars 'update: true' - dbt test --target bigquery - -test-all: test-postgres test-redshift test-snowflake test-bigquery - echo "Completed successfully" +test-all: $(TARGETS) + echo "Tested all targets" From 9acf4b21d00588aa08a3aa581c7d014f9840dd9e Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 20 Aug 2020 12:08:13 -0400 Subject: [PATCH 06/18] Update _is_ephemeral while we're here --- macros/cross_db_utils/_is_ephemeral.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/macros/cross_db_utils/_is_ephemeral.sql b/macros/cross_db_utils/_is_ephemeral.sql index e13be407..34c70db9 100644 --- a/macros/cross_db_utils/_is_ephemeral.sql +++ b/macros/cross_db_utils/_is_ephemeral.sql @@ -1,7 +1,8 @@ {% macro _is_ephemeral(obj, macro) %} {%- if obj.is_cte -%} - {% if obj.name.startswith('__dbt__CTE__') %} - {% set model_name = obj.name[12:] %} + {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %} + {% if obj.name.startswith(ephemeral_prefix) %} + {% set model_name = obj.name[(ephemeral_prefix|length):] %} {% else %} {% set model_name = obj.name %} {%- endif -%} From fecd4a40db7625a5cc29a7d2ab36863c29a9fdb2 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 20 Aug 2020 12:13:11 -0400 Subject: [PATCH 07/18] Lost & found: pg dateadd, datediff --- macros/cross_db_utils/dateadd.sql | 6 ++++++ macros/cross_db_utils/datediff.sql | 32 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 8df7fc77..e4a7eb43 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -22,3 +22,9 @@ ) {% endmacro %} + +{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %} + + {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) + +{% endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index 44c6052f..a4cb4d6a 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -24,3 +24,35 @@ {% endmacro %} +{% macro postgres__datediff(first_date, second_date, datepart) %} + + {% if datepart == 'year' %} + (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date)) + {% elif datepart == 'quarter' %} + ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) + {% elif datepart == 'month' %} + ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date)) + {% elif datepart == 'day' %} + (({{second_date}})::date - ({{first_date}})::date) + {% elif datepart == 'week' %} + ({{ dbt_utils.datediff(first_date, second_date, 'day') }} / 7 + case + when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then + case when {{first_date}} <= {{second_date}} then 0 else -1 end + else + case when {{first_date}} <= {{second_date}} then 1 else 0 end + end) + {% elif datepart == 'hour' %} + ({{ dbt_utils.datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp)) + {% elif datepart == 'minute' %} + ({{ dbt_utils.datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp)) + {% elif datepart == 'second' %} + ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp))) + {% elif datepart == 'millisecond' %} + ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp))) + {% elif datepart == 'microsecond' %} + ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp))) + {% else %} + {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }} + {% endif %} + +{% endmacro %} From 9760ba1e8ec596ffa9accb9a56da759b55626f36 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 21 Aug 2020 11:32:17 -0400 Subject: [PATCH 08/18] Add to readme, changelog --- CHANGELOG.md | 6 ++++++ README.md | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8648e52b..e5d73fb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ ## Features +* Switch usage of `adapter_macro` to `adapter.dispatch`, and define `dbt_utils_dispatch_list`, +enabling users of community-supported database plugins to add or override macro implementations +specific to their database (#267) +* Use `add_ephemeral_prefix` instead of hard-coding a string literal, to support +database adapters that use different prefixes (#267) + ## Quality of life # dbt-utils v0.5.1 diff --git a/README.md b/README.md index 30607d4e..0300d1cb 100644 --- a/README.md +++ b/README.md @@ -777,6 +777,42 @@ We welcome contributions to this repo! To contribute a new feature or a fix, ple ---- +### Dispatch macros + +**Note:** This is primarily relevant to users and maintainers of community-supported +database plugins. If you use Postgres, Redshift, Snowflake, or Bigquery, this likely +does not apply to you. + +dbt v0.18.0 introduces `adapter.dispatch()`, a reliable way to define different implementations of the same macro +across different databases. + +All dispatched macros in `dbt_utils` have an override setting: a `var` named +`dbt_utils_dispatch_list` that accepts a list of package names. If you set this +variable in your project, when dbt searches for implementations of a dispatched +`dbt_utils` macro, it will search through your listed packages _before_ using +the implementations defined in `dbt_utils`. + +Set the variable: +```yml +vars: + dbt_utils_dispatch_list: + - first_package_to_search # likely the name of your root project + - second_package_to_search # likely an "add-on" package, such as spark_utils + # dbt_utils is always the last place searched +``` + +When running on Spark, if dbt needs to dispatch `dbt_utils.datediff`, it will search for the following in order: +``` +first_package_to_search.spark__datediff +first_package_to_search.default__datediff +second_package_to_search.spark__datediff +second_package_to_search.default__datediff +dbt_utils.spark__datediff +dbt_utils.default__datediff +``` + +---- + ### Getting started with dbt - [What is dbt]? From 818d793d2a30511663623e76b501ab5bbeb71c23 Mon Sep 17 00:00:00 2001 From: Claire Carroll <20294432+clrcrl@users.noreply.github.com> Date: Thu, 20 Aug 2020 15:44:04 -0400 Subject: [PATCH 09/18] Remove get_tables_by_prefix and union_tables (deprecated in v0.2.4 / Dec 2019) --- README.md | 5 ----- .../sql/test_get_tables_by_prefix_and_union.sql | 4 ---- macros/sql/get_relations_by_prefix.sql | 12 ------------ macros/sql/union.sql | 12 ------------ 4 files changed, 33 deletions(-) delete mode 100644 integration_tests/models/sql/test_get_tables_by_prefix_and_union.sql diff --git a/README.md b/README.md index 0300d1cb..f5cb964e 100644 --- a/README.md +++ b/README.md @@ -442,9 +442,6 @@ Usage: ... ``` #### get_relations_by_prefix -> This replaces the `get_tables_by_prefix` macro. Note that the `get_tables_by_prefix` macro will -be deprecated in a future release of this package. - Returns a list of [Relations](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation) that match a given prefix, with an optional exclusion pattern. It's particularly handy paired with `union_relations`. @@ -516,8 +513,6 @@ from {{ref('my_model')}} ``` #### union_relations ([source](macros/sql/union.sql)) -> This replaces the `union_tables` macro. Note that the `union_tables` macro will -be deprecated in a future release of this package. This macro unions together an array of [Relations](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation), even when columns have differing orders in each Relation, and/or some columns are diff --git a/integration_tests/models/sql/test_get_tables_by_prefix_and_union.sql b/integration_tests/models/sql/test_get_tables_by_prefix_and_union.sql deleted file mode 100644 index 4ed6c21f..00000000 --- a/integration_tests/models/sql/test_get_tables_by_prefix_and_union.sql +++ /dev/null @@ -1,4 +0,0 @@ -{{config( materialized = 'table')}} - -{% set tables = dbt_utils.get_tables_by_prefix(target.schema, 'data_events_') %} -{{ dbt_utils.union_tables(tables) }} diff --git a/macros/sql/get_relations_by_prefix.sql b/macros/sql/get_relations_by_prefix.sql index 895ec795..3ae91d6f 100644 --- a/macros/sql/get_relations_by_prefix.sql +++ b/macros/sql/get_relations_by_prefix.sql @@ -21,15 +21,3 @@ {%- endif -%} {% endmacro %} - -{% macro get_tables_by_prefix(schema, prefix, exclude='', database=target.database) %} - {%- set error_message = ' - Warning: the `get_tables_by_prefix` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ - Use the `get_relations_by_prefix` macro instead. \ - The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} - {%- do exceptions.warn(error_message) -%} - - {{ return(dbt_utils.get_relations_by_prefix(schema, prefix, exclude, database)) }} - -{% endmacro %} diff --git a/macros/sql/union.sql b/macros/sql/union.sql index b6956f70..a5e04c01 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -83,15 +83,3 @@ {%- endfor -%} {%- endmacro -%} - -{%- macro union_tables(tables, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_table') -%} - {%- set error_message = ' - Warning: the `union_tables` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ - Use the `union_relations` macro instead. \ - The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} - {%- do exceptions.warn(error_message) -%} - - {{ return(dbt_utils.union_relations(tables, column_override, include, exclude, source_column_name)) }} - -{%- endmacro -%} From 13231765b331011b4c4488720970fbccaebe93f5 Mon Sep 17 00:00:00 2001 From: Claire Carroll <20294432+clrcrl@users.noreply.github.com> Date: Sat, 22 Aug 2020 16:20:08 -0400 Subject: [PATCH 10/18] Remove get_tables_by_pattern (erroneously introduced in v0.5.0) --- macros/sql/get_relations_by_pattern.sql | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index e09a6aba..0491452b 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -21,15 +21,3 @@ {%- endif -%} {% endmacro %} - -{% macro get_tables_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} - {%- set error_message = ' - Warning: the `get_tables_by_pattern` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ - Use the `get_relations_by_prefix` macro instead. \ - The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} - {%- do exceptions.warn(error_message) -%} - - {{ return(dbt_utils.get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database)) }} - -{% endmacro %} From 7ae385d9e5d8cee31917a8eb240cd5248e7e0011 Mon Sep 17 00:00:00 2001 From: Claire Carroll <20294432+clrcrl@users.noreply.github.com> Date: Sat, 22 Aug 2020 17:03:12 -0400 Subject: [PATCH 11/18] Update changelog --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5d73fb8..78cf5562 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # dbt-utils v0.6.0 (unreleased) +## Breaking changes +- :rotating_light: dbt v0.18.0 or greater is required for this release. If you are not ready to upgrade, consider using a previous release of this package +- :rotating_light: The `get_tables_by_prefix`, `union_tables` and `get_tables_by_pattern` macros have been removed + +## Migration instructions +- Upgrade your dbt project to v0.18.0 using [these instructions](https://discourse.getdbt.com/t/prerelease-v0-18-0-marian-anderson/1545). +- Upgrade your `dbt_project.yml` file to use version `0.7.0`. Run `dbt clean` and `dbt deps`. +- If your project uses the `get_tables_by_prefix` macro, replace it with `get_relations_by_prefix`. All arguments have retained the same name. +- If your project uses the `union_tables` macro, replace it with `union_relations`. While the order of arguments has stayed consistent, the `tables` argument has been renamed to `relations`. Further, the default value for the `source_column_name` argument has changed from `'_dbt_source_table'` to `'_dbt_source_relation'` — you may want to explicitly define this argument to avoid breaking changes. + +``` +-- before: +{{ dbt_utils.union_tables( + tables=[ref('my_model'), source('my_source', 'my_table')], + exclude=["_loaded_at"] +) }} + +-- after: +{{ dbt_utils.union_relations( + relations=[ref('my_model'), source('my_source', 'my_table')], + exclude=["_loaded_at"], + source_column_name='_dbt_source_table' +) }} +``` +- If your project uses the `get_tables_by_pattern` macro, replace it with `get_tables_by_pattern_sql` — all arguments are consistent. + ## Fixes ## Features @@ -11,6 +37,8 @@ specific to their database (#267) database adapters that use different prefixes (#267) ## Quality of life +* Remove deprecated macros `get_tables_by_prefix` and `union_tables` (#268) +* Remove `get_tables_by_pattern` macro, which is equivalent to the `get_tables_by_pattern_sql` macro (the latter has a more logical name) # dbt-utils v0.5.1 From 7aa80388f8a50a4ef9076a90dddabb5b119d54c4 Mon Sep 17 00:00:00 2001 From: Claire Carroll <20294432+clrcrl@users.noreply.github.com> Date: Sat, 22 Aug 2020 17:04:15 -0400 Subject: [PATCH 12/18] Remove crufty Jinja --- macros/sql/union.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/macros/sql/union.sql b/macros/sql/union.sql index a5e04c01..46bc22d2 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -1,4 +1,4 @@ -{%- macro union_relations(relations, column_override=none, include=[], exclude=[], source_column_name=none) -%} +{%- macro union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} {%- if exclude and include -%} {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }} @@ -10,7 +10,6 @@ {% endif -%} {%- set column_override = column_override if column_override is not none else {} -%} - {%- set source_column_name = source_column_name if source_column_name is not none else '_dbt_source_relation' -%} {%- set relation_columns = {} -%} {%- set column_superset = {} -%} From 12762a3ab185b528c2b3ad7ea2d925513ac04a8d Mon Sep 17 00:00:00 2001 From: "R. Joshua Huntley" Date: Fri, 28 Aug 2020 09:53:19 -0700 Subject: [PATCH 13/18] Upgrading schema test unique_comination_of_columns.sql to allow quoting of columns. --- .../unique_combination_of_columns.sql | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/macros/schema_tests/unique_combination_of_columns.sql b/macros/schema_tests/unique_combination_of_columns.sql index b7326470..733f4e86 100644 --- a/macros/schema_tests/unique_combination_of_columns.sql +++ b/macros/schema_tests/unique_combination_of_columns.sql @@ -1,8 +1,22 @@ -{% macro test_unique_combination_of_columns(model) %} +{% macro test_unique_combination_of_columns(model, quote_columns = false) %} {%- set columns = kwargs.get('combination_of_columns', kwargs.get('arg')) %} -{%- set columns_csv=columns | join(', ') %} +{% if not quote_columns %} + {%- set column_list=columns %} +{% elif quote_columns %} + {%- set column_list=[] %} + {% for column in columns -%} + {% set column_list = column_list.append( adapter.quote(column) ) %} + {%- endfor %} +{% else %} + {{ exceptions.raise_compiler_error( + "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" ~ quote ~"'.'" + ) }} +{% endif %} + +{%- set columns_csv=column_list | join(', ') %} + with validation_errors as ( From f2aad3a2914e2f36224be33df360f196ff5449cd Mon Sep 17 00:00:00 2001 From: "R. Joshua Huntley" Date: Fri, 28 Aug 2020 10:10:40 -0700 Subject: [PATCH 14/18] Adding info to changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78cf5562..4cd14b49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,3 +49,4 @@ database adapters that use different prefixes (#267) ## Quality of life * Improve release process, and fix tests (#251) * Make deprecation warnings more useful (#258 @tayloramurphy) +* Upgrade unique_combination_of_columns.sql macro to respect the quote_columns=TRUE flag as a test argument From 35048f6e45f61a558e9d06933da3df4e8be958e3 Mon Sep 17 00:00:00 2001 From: Claire Carroll <20294432+clrcrl@users.noreply.github.com> Date: Thu, 3 Sep 2020 13:12:03 -0400 Subject: [PATCH 15/18] Use released version of dbt --- run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_test.sh b/run_test.sh index 6bede664..2e1431be 100755 --- a/run_test.sh +++ b/run_test.sh @@ -6,7 +6,7 @@ if [[ ! -f $VENV ]]; then . $VENV pip install --upgrade pip setuptools - pip install "dbt==0.18.0rc1" + pip install "dbt>=0.18.0,<0.19.0" fi . $VENV From a203a8377f3afb3bec607c69a3150343f7869e8f Mon Sep 17 00:00:00 2001 From: Claire Carroll <20294432+clrcrl@users.noreply.github.com> Date: Thu, 3 Sep 2020 13:12:12 -0400 Subject: [PATCH 16/18] Update Changelog --- CHANGELOG.md | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cd14b49..92793251 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# dbt-utils v0.6.0 (unreleased) +# dbt-utils v0.6.0 ## Breaking changes - :rotating_light: dbt v0.18.0 or greater is required for this release. If you are not ready to upgrade, consider using a previous release of this package @@ -6,7 +6,7 @@ ## Migration instructions - Upgrade your dbt project to v0.18.0 using [these instructions](https://discourse.getdbt.com/t/prerelease-v0-18-0-marian-anderson/1545). -- Upgrade your `dbt_project.yml` file to use version `0.7.0`. Run `dbt clean` and `dbt deps`. +- Upgrade your `packages.yml` file to use version `0.6.0` of this package. Run `dbt clean` and `dbt deps`. - If your project uses the `get_tables_by_prefix` macro, replace it with `get_relations_by_prefix`. All arguments have retained the same name. - If your project uses the `union_tables` macro, replace it with `union_relations`. While the order of arguments has stayed consistent, the `tables` argument has been renamed to `relations`. Further, the default value for the `source_column_name` argument has changed from `'_dbt_source_table'` to `'_dbt_source_relation'` — you may want to explicitly define this argument to avoid breaking changes. @@ -26,27 +26,21 @@ ``` - If your project uses the `get_tables_by_pattern` macro, replace it with `get_tables_by_pattern_sql` — all arguments are consistent. -## Fixes - ## Features * Switch usage of `adapter_macro` to `adapter.dispatch`, and define `dbt_utils_dispatch_list`, enabling users of community-supported database plugins to add or override macro implementations -specific to their database (#267) +specific to their database ([#267](https://github.com/fishtown-analytics/dbt-utils/pull/267)) * Use `add_ephemeral_prefix` instead of hard-coding a string literal, to support -database adapters that use different prefixes (#267) +database adapters that use different prefixes ([#267](https://github.com/fishtown-analytics/dbt-utils/pull/267)) +* Implement a quote_columns argument in the unique_combination_of_columns schema test ([#270](https://github.com/fishtown-analytics/dbt-utils/pull/270) [@JoshuaHuntley](https://github.com/JoshuaHuntley)) ## Quality of life -* Remove deprecated macros `get_tables_by_prefix` and `union_tables` (#268) -* Remove `get_tables_by_pattern` macro, which is equivalent to the `get_tables_by_pattern_sql` macro (the latter has a more logical name) +* Remove deprecated macros `get_tables_by_prefix` and `union_tables` ([#268]((https://github.com/fishtown-analytics/dbt-utils/pull/268)) +* Remove `get_tables_by_pattern` macro, which is equivalent to the `get_tables_by_pattern_sql` macro (the latter has a more logical name) ([#268]((https://github.com/fishtown-analytics/dbt-utils/pull/268)) # dbt-utils v0.5.1 -## Fixes - -## Features - ## Quality of life -* Improve release process, and fix tests (#251) -* Make deprecation warnings more useful (#258 @tayloramurphy) -* Upgrade unique_combination_of_columns.sql macro to respect the quote_columns=TRUE flag as a test argument +* Improve release process, and fix tests ([#251](https://github.com/fishtown-analytics/dbt-utils/pull/251)) +* Make deprecation warnings more useful ([#258](https://github.com/fishtown-analytics/dbt-utils/pull/258)) [@tayloramurphy](https://github.com/tayloramurphy)) From bb75cc0039d0486ccbd0ff94c71d7b83d0fc67c2 Mon Sep 17 00:00:00 2001 From: Claire Carroll <20294432+clrcrl@users.noreply.github.com> Date: Thu, 3 Sep 2020 13:13:29 -0400 Subject: [PATCH 17/18] Update README --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f5cb964e..e52bc984 100644 --- a/README.md +++ b/README.md @@ -406,6 +406,18 @@ case we recommend using this test instead. - product ``` +An optional `quote_columns` parameter (`default=false`) can also be used if a column name needs to be quoted. + +```yaml +- name: revenue_by_product_by_month + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - month + - group + quote_columns: true +``` + --- ### SQL helpers #### get_query_results_as_dict ([source](macros/sql/get_query_results_as_dict.sql)) @@ -782,7 +794,7 @@ dbt v0.18.0 introduces `adapter.dispatch()`, a reliable way to define different across different databases. All dispatched macros in `dbt_utils` have an override setting: a `var` named -`dbt_utils_dispatch_list` that accepts a list of package names. If you set this +`dbt_utils_dispatch_list` that accepts a list of package names. If you set this variable in your project, when dbt searches for implementations of a dispatched `dbt_utils` macro, it will search through your listed packages _before_ using the implementations defined in `dbt_utils`. From dca02203b29fbac010440be61643b41387a11d0a Mon Sep 17 00:00:00 2001 From: Claire Carroll <20294432+clrcrl@users.noreply.github.com> Date: Thu, 3 Sep 2020 13:36:15 -0400 Subject: [PATCH 18/18] Fix small formatting issues in changelog [ci skip] --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92793251..2d4cf466 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,11 +36,11 @@ database adapters that use different prefixes ([#267](https://github.com/fishtow * Implement a quote_columns argument in the unique_combination_of_columns schema test ([#270](https://github.com/fishtown-analytics/dbt-utils/pull/270) [@JoshuaHuntley](https://github.com/JoshuaHuntley)) ## Quality of life -* Remove deprecated macros `get_tables_by_prefix` and `union_tables` ([#268]((https://github.com/fishtown-analytics/dbt-utils/pull/268)) -* Remove `get_tables_by_pattern` macro, which is equivalent to the `get_tables_by_pattern_sql` macro (the latter has a more logical name) ([#268]((https://github.com/fishtown-analytics/dbt-utils/pull/268)) +* Remove deprecated macros `get_tables_by_prefix` and `union_tables` ([#268](https://github.com/fishtown-analytics/dbt-utils/pull/268)) +* Remove `get_tables_by_pattern` macro, which is equivalent to the `get_tables_by_pattern_sql` macro (the latter has a more logical name) ([#268](https://github.com/fishtown-analytics/dbt-utils/pull/268)) # dbt-utils v0.5.1 ## Quality of life * Improve release process, and fix tests ([#251](https://github.com/fishtown-analytics/dbt-utils/pull/251)) -* Make deprecation warnings more useful ([#258](https://github.com/fishtown-analytics/dbt-utils/pull/258)) [@tayloramurphy](https://github.com/tayloramurphy)) +* Make deprecation warnings more useful ([#258](https://github.com/fishtown-analytics/dbt-utils/pull/258) [@tayloramurphy](https://github.com/tayloramurphy))