Skip to content

Commit 24bd9cb

Browse files
DVAlexHiggsDVAlexHiggs
committed
Release 0.10.2
2 parents f0cf634 + 7ae9c94 commit 24bd9cb

17 files changed

+176
-97
lines changed

‎dbt_project.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: automate_dv
2-
version: 0.10.1
2+
version: 0.10.2
33
require-dbt-version: [">=1.0.0", "<2.0.0"]
44
config-version: 2
55

‎macros/internal/metadata_processing/process_payload_column_excludes.sql

+5-4
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,18 @@
33
* This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
44
*/
55

6-
{%- macro process_payload_column_excludes(src_pk, src_hashdiff, src_payload, src_extra_columns,
6+
{%- macro process_payload_column_excludes(src_pk, src_hashdiff, src_payload, src_extra_columns, src_cdk,
77
src_eff, src_ldts, src_source, source_model) -%}
88

99
{%- if src_payload is not mapping -%}
1010
{%- do return(src_payload) -%}
1111
{%- endif -%}
1212

1313
{%- set source_model_cols = adapter.get_columns_in_relation(ref(source_model)) -%}
14-
{%- set columns_in_metadata = automate_dv.expand_column_list(columns=[src_pk, src_hashdiff,
15-
src_payload, src_extra_columns,
16-
src_eff, src_ldts, src_source]) | map('lower') | list -%}
14+
{%- set columns_in_metadata = automate_dv.expand_column_list(
15+
columns=[src_pk, src_hashdiff, src_cdk,
16+
src_payload, src_extra_columns,
17+
src_eff, src_ldts, src_source]) | map('lower') | list -%}
1718

1819
{%- set payload_cols = [] -%}
1920
{%- for col in source_model_cols -%}

‎macros/materialisations/error_messages.sql

+17
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,20 @@
8787
{%- endif -%}
8888

8989
{%- endmacro -%}
90+
91+
92+
{%- macro currently_disabled_error(func_name) -%}
93+
94+
{%- set message -%}
95+
This functionality ({{ func_name }}) is currently disabled for dbt-sqlserver 1.7.x,
96+
please revert to dbt-sqlserver 1.4.3 and AutomateDV 0.10.1 to use {{ func_name }}.
97+
98+
This is due to a suspected bug with the SQLServer Adapter in the 1.7.x version.
99+
We are actively working to get this fixed. Thank you for your understanding.
100+
{%- endset -%}
101+
102+
{%- if execute -%}
103+
{{- exceptions.raise_compiler_error(automate_dv.wrap_warning(message)) -}}
104+
{%- endif -%}
105+
106+
{%- endmacro -%}

‎macros/materialisations/period_mat_helpers/get_period_boundaries.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@
100100
CAST(COALESCE({{ automate_dv.timestamp_add(datepart, interval, from_date_or_timestamp) }},
101101
{{ current_timestamp() }} ) AS DATETIME2) AS stop_timestamp
102102
FROM {{ target_relation }}
103-
)
103+
)
104104
SELECT
105105
start_timestamp,
106106
stop_timestamp,

‎macros/materialisations/period_mat_helpers/get_period_filter_sql.sql

-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434

3535

3636
{% macro sqlserver__get_period_filter_sql(target_cols_csv, base_sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%}
37-
3837
{%- set filtered_sql = {'sql': base_sql} -%}
3938

4039
{%- do filtered_sql.update({'sql': automate_dv.replace_placeholder_with_period_filter(core_sql=filtered_sql.sql,

‎macros/materialisations/period_mat_helpers/replace_placeholder_with_period_filter.sql

+13-9
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,11 @@
1919
{% macro default__replace_placeholder_with_period_filter(core_sql, timestamp_field, start_timestamp, stop_timestamp, offset, period) %}
2020

2121
{%- set period_filter -%}
22-
(TO_TIMESTAMP({{ timestamp_field }})
23-
>= DATE_TRUNC('{{ period }}', TO_TIMESTAMP('{{ start_timestamp }}') + INTERVAL '{{ offset }} {{ period }}') AND
24-
TO_TIMESTAMP({{ timestamp_field }}) < DATE_TRUNC('{{ period }}', TO_TIMESTAMP('{{ start_timestamp }}') + INTERVAL '{{ offset }} {{ period }}' + INTERVAL '1 {{ period }}'))
25-
AND (TO_TIMESTAMP({{ timestamp_field }}) >= TO_TIMESTAMP('{{ start_timestamp }}'))
22+
(
23+
TO_TIMESTAMP({{ timestamp_field }}) >= DATE_TRUNC('{{ period }}', TO_TIMESTAMP('{{ start_timestamp }}') + INTERVAL '{{ offset }} {{ period }}')
24+
AND TO_TIMESTAMP({{ timestamp_field }}) < DATE_TRUNC('{{ period }}', TO_TIMESTAMP('{{ start_timestamp }}') + INTERVAL '{{ offset }} {{ period }}' + INTERVAL '1 {{ period }}'))
25+
AND (TO_TIMESTAMP({{ timestamp_field }}) >= TO_TIMESTAMP('{{ start_timestamp }}')
26+
)
2627
{%- endset -%}
2728
{%- set filtered_sql = core_sql | replace("__PERIOD_FILTER__", period_filter) -%}
2829

@@ -58,10 +59,13 @@
5859

5960
{# MSSQL cannot CAST datetime2 strings with more than 7 decimal places #}
6061
{% set start_timestamp_mssql = start_timestamp[0:27] %}
62+
6163
{%- set period_filter -%}
62-
(CAST({{ timestamp_field }} AS DATETIME2) >= DATEADD({{ period }}, DATEDIFF({{ period }}, 0, DATEADD({{ period }}, {{ offset }}, CAST('{{ start_timestamp_mssql }}' AS DATETIME2))), 0) AND
63-
CAST({{ timestamp_field }} AS DATETIME2) < DATEADD({{ period }}, 1, DATEADD({{ period }}, {{ offset }}, CAST('{{ start_timestamp_mssql }}' AS DATETIME2)))
64-
AND (CAST({{ timestamp_field }} AS DATETIME2) >= CAST('{{ start_timestamp_mssql }}' AS DATETIME2)))
64+
(
65+
CAST({{ timestamp_field }} AS DATETIME2) >= DATEADD({{ period }}, DATEDIFF({{ period }}, 0, DATEADD({{ period }}, {{ offset }}, CAST('{{ start_timestamp_mssql }}' AS DATETIME2))), 0)
66+
AND CAST({{ timestamp_field }} AS DATETIME2) < DATEADD({{ period }}, 1, DATEADD({{ period }}, {{ offset }}, CAST('{{ start_timestamp_mssql }}' AS DATETIME2)))
67+
AND (CAST({{ timestamp_field }} AS DATETIME2) >= CAST('{{ start_timestamp_mssql }}' AS DATETIME2))
68+
)
6569
{%- endset -%}
6670

6771
{%- set filtered_sql = core_sql | replace("__PERIOD_FILTER__", period_filter) -%}
@@ -74,8 +78,8 @@
7478

7579
{%- set period_filter -%}
7680
{{ timestamp_field }}::TIMESTAMP >= DATE_TRUNC('{{ period }}', TIMESTAMP '{{ start_timestamp }}' + INTERVAL '{{ offset }} {{ period }}')
77-
AND {{ timestamp_field }}::TIMESTAMP < DATE_TRUNC('{{ period }}', TIMESTAMP '{{ start_timestamp }}' + INTERVAL '{{ offset }} {{ period }}' + INTERVAL '1 {{ period }}')
78-
AND {{ timestamp_field }}::TIMESTAMP >= TIMESTAMP '{{ start_timestamp }}'
81+
AND {{ timestamp_field }}::TIMESTAMP < DATE_TRUNC('{{ period }}', TIMESTAMP '{{ start_timestamp }}' + INTERVAL '{{ offset }} {{ period }}' + INTERVAL '1 {{ period }}')
82+
AND {{ timestamp_field }}::TIMESTAMP >= TIMESTAMP '{{ start_timestamp }}'
7983
{%- endset -%}
8084
{%- set filtered_sql = core_sql | replace("__PERIOD_FILTER__", period_filter) -%}
8185

‎macros/materialisations/vault_insert_by_period_materialization.sql

+17-22
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,7 @@
1818

1919
{{ automate_dv.experimental_not_recommended_warning(func_name='vault_insert_by_period') }}
2020

21-
{% if target.type == "sqlserver" %}
22-
{%- set target_relation = this.incorporate(type='table') -%}
23-
{% else %}
24-
{%- set target_relation = this -%}
25-
{% endif %}
26-
21+
{%- set target_relation = this.incorporate(type='table') -%}
2722
{%- set existing_relation = load_relation(this) -%}
2823
{%- set tmp_relation = make_temp_relation(target_relation) -%}
2924

@@ -44,7 +39,6 @@
4439
{{ run_hooks(pre_hooks, inside_transaction=True) }}
4540

4641
{% if existing_relation is none %}
47-
4842
{% set filtered_sql = automate_dv.replace_placeholder_with_period_filter(core_sql=sql, timestamp_field=timestamp_field,
4943
start_timestamp=start_stop_dates.start_date,
5044
stop_timestamp=start_stop_dates.stop_date,
@@ -53,7 +47,6 @@
5347
{% do to_drop.append(tmp_relation) %}
5448

5549
{% elif existing_relation.is_view %}
56-
5750
{{ log("Dropping relation " ~ target_relation ~ " because it is a view and this model is a table (vault_insert_by_period).") }}
5851
{% do adapter.drop_relation(existing_relation) %}
5952
{% set build_sql = create_table_as(False, target_relation, filtered_sql) %}
@@ -66,21 +59,19 @@
6659

6760
{% elif full_refresh_mode %}
6861
{% set filtered_sql = automate_dv.replace_placeholder_with_period_filter(core_sql=sql, timestamp_field=timestamp_field,
69-
start_timestamp=start_stop_dates.start_date,
70-
stop_timestamp=start_stop_dates.stop_date,
71-
offset=0, period=period) %}
62+
start_timestamp=start_stop_dates.start_date,
63+
stop_timestamp=start_stop_dates.stop_date,
64+
offset=0, period=period) %}
7265
{% if target.type in ['postgres', 'sqlserver'] %}
7366
{{ automate_dv.drop_temporary_special(target_relation) }}
7467
{% endif %}
7568

7669
{% set build_sql = create_table_as(False, target_relation, filtered_sql) %}
7770
{% else %}
78-
{% set period_boundaries = automate_dv.get_period_boundaries(target_relation,
79-
timestamp_field,
80-
start_stop_dates.start_date,
81-
start_stop_dates.stop_date,
82-
period) %}
83-
71+
{% set period_boundaries = automate_dv.get_period_boundaries(target_relation, timestamp_field,
72+
start_stop_dates.start_date,
73+
start_stop_dates.stop_date,
74+
period) %}
8475
{% set target_columns = adapter.get_columns_in_relation(target_relation) %}
8576
{%- set target_cols_csv = target_columns | map(attribute='quoted') | join(', ') -%}
8677
{%- set loop_vars = {'sum_rows_inserted': 0} -%}
@@ -96,10 +87,8 @@
9687
{% set tmp_relation = make_temp_relation(target_relation) %}
9788

9889
{% set tmp_table_sql = automate_dv.get_period_filter_sql(target_cols_csv, sql, timestamp_field, period,
99-
period_boundaries.start_timestamp,
100-
period_boundaries.stop_timestamp, i) %}
101-
102-
90+
period_boundaries.start_timestamp,
91+
period_boundaries.stop_timestamp, i) %}
10392

10493
{# This call statement drops and then creates a temporary table #}
10594
{# but MSSQL will fail to drop any temporary table created by a previous loop iteration #}
@@ -211,4 +200,10 @@
211200

212201
{{ return({'relations': [target_relation]}) }}
213202

214-
{%- endmaterialization %}
203+
{%- endmaterialization %}
204+
205+
{% materialization vault_insert_by_period, adapter='sqlserver' %}
206+
207+
{{ automate_dv.currently_disabled_error(func_name='vault_insert_by_period') }}
208+
209+
{% endmaterialization %}

‎macros/materialisations/vault_insert_by_rank_materialization.sql

+7-1
Original file line numberDiff line numberDiff line change
@@ -166,4 +166,10 @@
166166

167167
{{ return({'relations': [target_relation]}) }}
168168

169-
{%- endmaterialization %}
169+
{%- endmaterialization %}
170+
171+
{% materialization vault_insert_by_rank, adapter='sqlserver' %}
172+
173+
{{ automate_dv.currently_disabled_error(func_name='vault_insert_by_rank') }}
174+
175+
{% endmaterialization %}

‎macros/supporting/as_of_date_window.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ new_rows_as_of AS (
7979
WHERE a.AS_OF_DATE >= (SELECT LAST_SAFE_LOAD_DATETIME FROM last_safe_load_datetime)
8080
UNION
8181
{%- endif %}
82-
SELECT as_of_date
82+
SELECT AS_OF_DATE
8383
FROM as_of_grain_new_entries
8484
),
8585

‎macros/supporting/data_types/type_binary.sql

+2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
BINARY(16)
1313
{%- elif var('hash', 'MD5') | lower == 'sha' -%}
1414
BINARY(32)
15+
{%- elif var('hash', 'MD5') | lower == 'sha1' -%}
16+
BINARY(20)
1517
{%- else -%}
1618
BINARY(16)
1719
{%- endif -%}

‎macros/supporting/data_types/type_string.sql

+2
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
VARCHAR(16)
2626
{%- elif var('hash', 'MD5') | lower == 'sha' -%}
2727
VARCHAR(32)
28+
{%- elif var('hash', 'MD5') | lower == 'sha1' -%}
29+
VARCHAR(20)
2830
{%- endif -%}
2931
{%- else -%}
3032
VARCHAR({{ char_length }})

‎macros/supporting/ghost_records/binary_ghost.sql

+4
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
{{ automate_dv.cast_binary(column_str=modules.itertools.repeat('0', 32) | join (''), alias=alias, quote=true) }}
1414
{%- elif hash | lower == 'sha' -%}
1515
{{ automate_dv.cast_binary(column_str=modules.itertools.repeat('0', 64) | join (''), alias=alias, quote=true) }}
16+
{%- elif hash | lower == 'sha1' -%}
17+
{{ automate_dv.cast_binary(column_str=modules.itertools.repeat('0', 40) | join (''), alias=alias, quote=true) }}
1618
{%- else -%}
1719
{{ automate_dv.cast_binary(column_str=modules.itertools.repeat('0', 32) | join (''), alias=alias, quote=true) }}
1820
{%- endif -%}
@@ -23,6 +25,8 @@
2325
CAST(REPLICATE(CAST(CAST('0' AS tinyint) AS BINARY(16)), 16) AS BINARY(16))
2426
{%- elif hash | lower == 'sha' -%}
2527
CAST(REPLICATE(CAST(CAST('0' AS tinyint) AS BINARY(32)), 32) AS BINARY(32))
28+
{%- elif hash | lower == 'sha1' -%}
29+
CAST(REPLICATE(CAST(CAST('0' AS tinyint) AS BINARY(20)), 20) AS BINARY(20))
2630
{%- else -%}
2731
CAST(REPLICATE(CAST(CAST('0' AS tinyint) AS BINARY(16)), 16) AS BINARY(16))
2832
{%- endif -%}

‎macros/supporting/hash_components/select_hash_alg.sql

+44-2
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,18 @@
55

66
{%- macro select_hash_alg(hash) -%}
77

8-
{%- set available_hash_algorithms = ['md5', 'sha'] -%}
8+
{%- set available_hash_algorithms = ['md5', 'sha', 'sha1'] -%}
99

1010
{%- if execute and hash | lower not in available_hash_algorithms %}
11-
{%- do exceptions.warn("Configured hash ('{}') not recognised. Must be one of: {} (case insensitive)".format(hash | lower, available_hash_algorithms | join(', '))) -%}
11+
{%- do exceptions.warn("Configured hash ('{}') not recognised. Must be one of: {} (case insensitive). Defaulting to MD5 hashing.".format(hash | lower, available_hash_algorithms | join(', '))) -%}
1212
{%- endif -%}
1313

1414
{%- if hash | lower == 'md5' -%}
1515
{%- do return(automate_dv.hash_alg_md5()) -%}
1616
{%- elif hash | lower == 'sha' -%}
1717
{%- do return(automate_dv.hash_alg_sha256()) -%}
18+
{%- elif hash | lower == 'sha1' -%}
19+
{%- do return(automate_dv.hash_alg_sha1()) -%}
1820
{%- else -%}
1921
{%- do return(automate_dv.hash_alg_md5()) -%}
2022
{%- endif -%}
@@ -102,3 +104,43 @@
102104
{% do return('UPPER(SHA2([HASH_STRING_PLACEHOLDER], 256))') %}
103105

104106
{% endmacro %}
107+
108+
{#- SHA1 -#}
109+
110+
{%- macro hash_alg_sha1() -%}
111+
112+
{{- adapter.dispatch('hash_alg_sha1', 'automate_dv')() -}}
113+
114+
{%- endmacro %}
115+
116+
{% macro default__hash_alg_sha1() -%}
117+
118+
{% do return(automate_dv.cast_binary('SHA1_BINARY([HASH_STRING_PLACEHOLDER])', quote=false)) %}
119+
120+
{% endmacro %}
121+
122+
{% macro bigquery__hash_alg_sha1() -%}
123+
124+
{% do return(automate_dv.cast_binary('UPPER(TO_HEX(SHA1([HASH_STRING_PLACEHOLDER])))', quote=false)) %}
125+
126+
{% endmacro %}
127+
128+
{% macro sqlserver__hash_alg_sha1() -%}
129+
130+
{% do return(automate_dv.cast_binary("HASHBYTES('SHA1', [HASH_STRING_PLACEHOLDER])", quote=false)) %}
131+
132+
{% endmacro %}
133+
134+
{% macro postgres__hash_alg_sha1() -%}
135+
136+
{%- do exceptions.warn("Configured hash (SHA-1) is not supported on Postgres.
137+
Defaulting to hash 'MD5', alternatively configure your hash as 'SHA' for SHA256 hashing.") -%}
138+
{{ automate_dv.hash_alg_md5() }}
139+
140+
{% endmacro %}
141+
142+
{% macro databricks__hash_alg_sha1() -%}
143+
144+
{% do return('UPPER(SHA1([HASH_STRING_PLACEHOLDER]))') %}
145+
146+
{% endmacro %}

0 commit comments

Comments
 (0)