Fix profile_where_clause tests and profile_over_time in BigQuery …

…+ Snowflake (#58) * Fix typos in profile_where_clause.yml * Fix profile_where_clause tests * Fix where_clause position in get_profile macro * Fix profile_over_time in bigquery * Fix profile_over_time in snowflake * Add Databricks to PR template Co-authored-by: Simo Tumelius <[email protected]>
data-mie · Nov 15, 2022 · 48497d9 · 48497d9
1 parent 83bb2a5
commit 48497d9
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 17 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -10,5 +10,6 @@ Describe your changes, and why you're making them.
     - [ ] Snowflake
     - [ ] Redshift
     - [ ] SQL Server
+    - [ ] Databricks
 - [ ] I have written tests for new macros (either as dbt schema tests in `integration_tests/models`, dbt data tests in `integration_tests/tests` or integration tests in the [CI workflow](workflows/main.yml))
 - [ ] I have updated the README.md (if applicable)
diff --git a/integration_tests/models/profile_over_time.sql b/integration_tests/models/profile_over_time.sql
@@ -1,6 +1,14 @@
+{%- set incremental_strategy=none -%}
+{%- if target.type == "bigquery" -%}
+  {%- set incremental_strategy = "merge" -%}
+{%- elif target.type == "snowflake" -%}
+  {%- set incremental_strategy = "merge" -%}
+{%- endif -%}
+
 {{
   config(
-    materialized="incremental"
+    materialized="incremental",
+    incremental_strategy=incremental_strategy
   )
 }}
 

diff --git a/integration_tests/models/profile_where_clause.sql b/integration_tests/models/profile_where_clause.sql
@@ -1,8 +1,5 @@
 -- depends_on: {{ ref("test_data") }}
 {% if execute %}
   {%- set where_clause = "string_not_nullable = 'one'" -%}
-  {%- if target.type == "snowflake" -%}
-    {%- set where_clause = where_clause -%}
-  {%- endif -%}
   {{ dbt_profiler.get_profile(relation=ref("test_data"), where_clause=where_clause) }}
 {% endif %}
diff --git a/integration_tests/models/profile_where_clause.yml b/integration_tests/models/profile_where_clause.yml
@@ -1,9 +1,9 @@
 version: 2
 
 models:
-  - name: profile_exclude_columns
+  - name: profile_where_clause
     columns:
-      - name: string_not_nullable
+      - name: row_count
         tests:
-          - dbt_expectations.expect_column_values_to_match_like_pattern:
-              like_patter: ["one"]
+          - dbt_utils.expression_is_true:
+              expression: "= 2"
diff --git a/macros/get_profile.sql b/macros/get_profile.sql
@@ -60,6 +60,9 @@
       select
         *
       from {{ relation }}
+      {% if where_clause %}
+        where {{ where_clause }}
+      {% endif %}
     ),
 
     column_profiles as (
@@ -101,9 +104,6 @@
           cast(current_timestamp as {{ dbt_profiler.type_string() }}) as profiled_at,
           {{ loop.index }} as _column_position
         from source_data
-        {% if where_clause %}
-        where {{ where_clause }}
-        {% endif %}
 
         {% if not loop.last %}union all{% endif %}
       {% endfor %}
@@ -188,6 +188,9 @@
       select
         *
       from {{ relation }}
+      {% if where_clause %}
+        where {{ where_clause }}
+      {% endif %}
     ),
 
     column_profiles as (
@@ -229,9 +232,6 @@
           cast(current_timestamp as {{ dbt_profiler.type_string() }}) as profiled_at,
           {{ loop.index }} as _column_position
         from source_data
-        {% if where_clause %}
-        where {{ where_clause }}
-        {% endif %}
         {% if not loop.last %}union all{% endif %}
       {% endfor %}
     )
@@ -312,6 +312,9 @@
       select
         *
       from {{ relation }}
+      {% if where_clause %}
+        where {{ where_clause }}
+      {% endif %}
     ),
 
     column_profiles as (
@@ -353,9 +356,6 @@
           cast(current_timestamp as {{ dbt_profiler.type_string() }}) as profiled_at,
           {{ loop.index }} as _column_position
         from source_data
-        {% if where_clause %}
-        where {{ where_clause }}
-        {% endif %}
 
         {% if not loop.last %}union all{% endif %}
       {% endfor %}