Fix median calculation in Redshift (#89)

* Fix. Issue 84. Median function fixed for Redshift * Fix. all median calculations need to include the 3rd parameter
data-mie · Jun 10, 2024 · 39f3e48 · 39f3e48
1 parent 4a76288
commit 39f3e48
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 9 deletions.
diff --git a/macros/get_profile.sql b/macros/get_profile.sql
@@ -100,7 +100,7 @@
             {{ dbt_profiler.measure_avg(column_name, data_type) }} as avg,
           {%- endif %}
           {% if "median" not in exclude_measures -%}
-            {{ dbt_profiler.measure_median(column_name, data_type) }} as median,
+            ({{ dbt_profiler.measure_median(column_name, data_type, 'source_data') }}) as median,
           {%- endif %}
           {% if "std_dev_population" not in exclude_measures -%}
             {{ dbt_profiler.measure_std_dev_population(column_name, data_type) }} as std_dev_population,
@@ -240,7 +240,7 @@
             {{ dbt_profiler.measure_avg(column_name, data_type) }} as avg,
           {%- endif %}
           {% if "median" not in exclude_measures -%}
-            {{ dbt_profiler.measure_median(column_name, data_type) }} as median,
+            ({{ dbt_profiler.measure_median(column_name, data_type, 'source_data') }}) as median,
           {%- endif %}
           {% if "std_dev_population" not in exclude_measures -%}
             {{ dbt_profiler.measure_std_dev_population(column_name, data_type) }} as std_dev_population,
@@ -377,7 +377,7 @@
             {{ dbt_profiler.measure_avg(column_name, data_type) }} as avg,
           {%- endif %}
           {% if "median" not in exclude_measures -%}
-            {{ dbt_profiler.measure_median(column_name, data_type) }} as median,
+            ({{ dbt_profiler.measure_median(column_name, data_type, 'source_data') }}) as median,
           {%- endif %}
           {% if "std_dev_population" not in exclude_measures -%}
             {{ dbt_profiler.measure_std_dev_population(column_name, data_type) }} as std_dev_population,

diff --git a/macros/measures.sql b/macros/measures.sql
@@ -129,11 +129,11 @@ case when count(distinct {{ adapter.quote(column_name) }}) = count(*) then 1 els
 
 {# measure_median  -------------------------------------------------     #}
 
-{%- macro measure_median(column_name, data_type) -%}
-  {{ return(adapter.dispatch("measure_median", macro_namespace="dbt_profiler")(column_name, data_type)) }}
+{%- macro measure_median(column_name, data_type, cte_name) -%}
+  {{ return(adapter.dispatch("measure_median", macro_namespace="dbt_profiler")(column_name, data_type, cte_name)) }}
 {%- endmacro -%}
 
-{%- macro default__measure_median(column_name, data_type) -%}
+{%- macro default__measure_median(column_name, data_type, cte_name) -%}
 
 {%- if dbt_profiler.is_numeric_dtype(data_type) and not dbt_profiler.is_struct_dtype(data_type) -%}
     median({{ adapter.quote(column_name) }})
@@ -143,7 +143,7 @@ case when count(distinct {{ adapter.quote(column_name) }}) = count(*) then 1 els
 
 {%- endmacro -%}
 
-{%- macro bigquery__measure_median(column_name, data_type) -%}
+{%- macro bigquery__measure_median(column_name, data_type, cte_name) -%}
 
 {%- if dbt_profiler.is_numeric_dtype(data_type) and not dbt_profiler.is_struct_dtype(data_type) -%}
     APPROX_QUANTILES({{ adapter.quote(column_name) }}, 100)[OFFSET(50)]
@@ -153,7 +153,7 @@ case when count(distinct {{ adapter.quote(column_name) }}) = count(*) then 1 els
 
 {%- endmacro -%}
 
-{%- macro postgres__measure_median(column_name, data_type) -%}
+{%- macro postgres__measure_median(column_name, data_type, cte_name) -%}
 
 {%- if dbt_profiler.is_numeric_dtype(data_type) and not dbt_profiler.is_struct_dtype(data_type) -%}
     percentile_cont(0.5) within group (order by {{ adapter.quote(column_name) }})
@@ -163,7 +163,17 @@ case when count(distinct {{ adapter.quote(column_name) }}) = count(*) then 1 els
 
 {%- endmacro -%}
 
-{%- macro sql_server__measure_median(column_name, data_type) -%}
+{%- macro redshift__measure_median(column_name, data_type, cte_name) -%}
+
+{%- if dbt_profiler.is_numeric_dtype(data_type) and not dbt_profiler.is_struct_dtype(data_type) -%}
+    select percentile_cont(0.5) within group (order by {{ adapter.quote(column_name) }}) from {{ cte_name }}
+{%- else -%}
+    cast(null as {{ dbt.type_numeric() }})
+{%- endif -%}
+
+{%- endmacro -%}
+
+{%- macro sql_server__measure_median(column_name, data_type, cte_name) -%}
 
 {%- if dbt_profiler.is_numeric_dtype(data_type) and not dbt_profiler.is_struct_dtype(data_type) -%}
     percentile_cont({{ adapter.quote(column_name) }}, 0.5) over ()