Handle schema pattern on BQ

BP127 · Sep 4, 2020 · c02a04e · c02a04e
1 parent 6a9aa6f
commit c02a04e
Showing 1 changed file with 46 additions and 8 deletions.
diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql
@@ -5,7 +5,7 @@
 
 {% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %}
 
-        select distinct 
+        select distinct
             table_schema as "table_schema", table_name as "table_name"
         from {{database}}.information_schema.tables
         where table_schema ilike '{{ schema_pattern }}'
@@ -16,13 +16,51 @@
 
 
 {% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %}
-
-        select distinct
-            table_schema, table_name
 
-        from {{adapter.quote(database)}}.{{schema}}.INFORMATION_SCHEMA.TABLES
-        where table_schema = '{{schema_pattern}}'
-            and lower(table_name) like lower ('{{table_pattern}}')
-            and lower(table_name) not like lower ('{{exclude}}')
+    {% if '%' in schema_pattern %}
+        {% set schemata=dbt_utils._bigquery__get_matching_schemata(schema_pattern, database) %}
+    {% else %}
+        {% set schemata=[schema_pattern] %}
+    {% endif %}
+
+    {% set sql %}
+        {% for schema in schemata %}
+            select distinct
+                table_schema, table_name
+
+            from {{ adapter.quote(database) }}.{{ schema }}.INFORMATION_SCHEMA.TABLES
+            where lower(table_name) like lower ('{{ table_pattern }}')
+                and lower(table_name) not like lower ('{{ exclude }}')
+
+            {% if not loop.last %} union all {% endif %}
+
+        {% endfor %}
+    {% endset %}
+
+    {{ return(sql) }}
+
+{% endmacro %}
+
+
+{% macro _bigquery__get_matching_schemata(schema_pattern, database) %}
+    {% if execute %}
+
+        {% set sql %}
+        select schema_name from {{ adapter.quote(database) }}.INFORMATION_SCHEMA.SCHEMATA
+        where lower(schema_name) like lower('{{ schema_pattern }}')
+        {% endset %}
+
+        {% set results=run_query(sql) %}
+
+        {% set schemata=results.columns['schema_name'].values() %}
+
+        {{ return(schemata) }}
+
+    {% else %}
+
+        {{ return([]) }}
+
+    {% endif %}
+
 
 {% endmacro %}