Skip to content

Commit f0cf634

Browse files
DVAlexHiggsDVAlexHiggs
committed
Release 0.10.1
2 parents 838522a + db10154 commit f0cf634

File tree

3 files changed

+60
-9
lines changed

3 files changed

+60
-9
lines changed

‎dbt_project.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: automate_dv
2-
version: 0.10.0
2+
version: 0.10.1
33
require-dbt-version: [">=1.0.0", "<2.0.0"]
44
config-version: 2
55

‎macros/tables/postgres/sat.sql

+29-3
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55

66
{%- macro postgres__sat(src_pk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%}
77

8+
{%- set apply_source_filter = config.get('apply_source_filter', false) -%}
9+
{%- set enable_ghost_record = var('enable_ghost_records', false) %}
10+
811
{%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source]) -%}
912
{%- set window_cols = automate_dv.expand_column_list(columns=[src_pk, src_hashdiff, src_ldts]) -%}
1013
{%- set pk_cols = automate_dv.expand_column_list(columns=[src_pk]) -%}
11-
{%- set enable_ghost_record = var('enable_ghost_records', false) -%}
1214

1315
{%- if model.config.materialized == 'vault_insert_by_rank' %}
1416
{%- set source_cols_with_rank = source_cols + [config.get('rank_column')] -%}
@@ -49,6 +51,21 @@ latest_records AS (
4951
WHERE b.rank = 1
5052
),
5153

54+
{%- if apply_source_filter %}
55+
56+
valid_stg AS (
57+
SELECT {{ automate_dv.prefix(source_cols, 's', alias_target='source') }}
58+
FROM source_data AS s
59+
LEFT JOIN latest_records AS sat
60+
ON {{ automate_dv.multikey(src_pk, prefix=['s', 'sat'], condition='=') }}
61+
WHERE {{ automate_dv.multikey(src_pk, prefix='sat', condition='IS NULL') }}
62+
OR {{ automate_dv.prefix([src_ldts], 's') }} > (
63+
SELECT MAX({{ src_ldts }}) FROM latest_records AS sat
64+
WHERE {{ automate_dv.multikey(src_pk, prefix=['sat','s'], condition='=') }}
65+
)
66+
),
67+
{%- endif %}
68+
5269
{%- endif %}
5370

5471
first_record_in_set AS (
@@ -59,7 +76,12 @@ first_record_in_set AS (
5976
PARTITION BY {{ automate_dv.prefix([src_pk], 'sd', alias_target='source') }}
6077
ORDER BY {{ automate_dv.prefix([src_ldts], 'sd', alias_target='source') }} ASC
6178
) as asc_rank
62-
FROM source_data as sd ) rin
79+
{%- if automate_dv.is_any_incremental() and apply_source_filter %}
80+
FROM valid_stg as sd
81+
{%- else %}
82+
FROM source_data as sd
83+
{%- endif %}
84+
) AS rin
6385
WHERE rin.asc_rank = 1
6486
),
6587

@@ -72,8 +94,12 @@ unique_source_records AS (
7294
LAG({{ automate_dv.prefix([src_hashdiff], 'sd', alias_target='source') }}) OVER (
7395
PARTITION BY {{ automate_dv.prefix([src_pk], 'sd', alias_target='source') }}
7496
ORDER BY {{ automate_dv.prefix([src_ldts], 'sd', alias_target='source') }} ASC) as prev_hashdiff
97+
{%- if automate_dv.is_any_incremental() and apply_source_filter %}
98+
FROM valid_stg as sd
99+
{%- else %}
75100
FROM source_data as sd
76-
) b
101+
{%- endif %}
102+
) AS b
77103
WHERE {{ automate_dv.prefix([src_hashdiff], 'b', alias_target='source') }} != b.prev_hashdiff
78104
),
79105

‎macros/tables/snowflake/sat.sql

+30-5
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,12 @@
2525

2626
{%- macro default__sat(src_pk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%}
2727

28+
{%- set apply_source_filter = config.get('apply_source_filter', false) -%}
29+
{%- set enable_ghost_record = var('enable_ghost_records', false) %}
30+
2831
{%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source]) -%}
2932
{%- set window_cols = automate_dv.expand_column_list(columns=[src_pk, src_hashdiff, src_ldts]) -%}
3033
{%- set pk_cols = automate_dv.expand_column_list(columns=[src_pk]) -%}
31-
{%- set enable_ghost_record = var('enable_ghost_records', false) %}
3234

3335
{%- if model.config.materialized == 'vault_insert_by_rank' %}
3436
{%- set source_cols_with_rank = source_cols + [config.get('rank_column')] -%}
@@ -66,6 +68,21 @@ latest_records AS (
6668
QUALIFY rank_num = 1
6769
),
6870

71+
{%- if apply_source_filter %}
72+
73+
valid_stg AS (
74+
SELECT {{ automate_dv.prefix(source_cols, 's', alias_target='source') }}
75+
FROM source_data AS s
76+
LEFT JOIN latest_records AS sat
77+
ON {{ automate_dv.multikey(src_pk, prefix=['s', 'sat'], condition='=') }}
78+
WHERE {{ automate_dv.multikey(src_pk, prefix='sat', condition='IS NULL') }}
79+
OR {{ automate_dv.prefix([src_ldts], 's') }} > (
80+
SELECT MAX({{ src_ldts }}) FROM latest_records AS sat
81+
WHERE {{ automate_dv.multikey(src_pk, prefix=['sat','s'], condition='=') }}
82+
)
83+
),
84+
{%- endif %}
85+
6986
{%- endif %}
7087

7188
first_record_in_set AS (
@@ -75,14 +92,22 @@ first_record_in_set AS (
7592
PARTITION BY {{ automate_dv.prefix([src_pk], 'sd', alias_target='source') }}
7693
ORDER BY {{ automate_dv.prefix([src_ldts], 'sd', alias_target='source') }} ASC
7794
) as asc_rank
95+
{%- if automate_dv.is_any_incremental() and apply_source_filter %}
96+
FROM valid_stg as sd
97+
{%- else %}
7898
FROM source_data as sd
99+
{%- endif %}
79100
QUALIFY asc_rank = 1
80101
),
81102

82103
unique_source_records AS (
83104
SELECT DISTINCT
84105
{{ automate_dv.prefix(source_cols, 'sd', alias_target='source') }}
106+
{%- if automate_dv.is_any_incremental() and apply_source_filter %}
107+
FROM valid_stg as sd
108+
{%- else %}
85109
FROM source_data as sd
110+
{%- endif %}
86111
QUALIFY {{ automate_dv.prefix([src_hashdiff], 'sd', alias_target='source') }} != LAG({{ automate_dv.prefix([src_hashdiff], 'sd', alias_target='source') }}) OVER (
87112
PARTITION BY {{ automate_dv.prefix([src_pk], 'sd', alias_target='source') }}
88113
ORDER BY {{ automate_dv.prefix([src_ldts], 'sd', alias_target='source') }} ASC)
@@ -93,9 +118,9 @@ unique_source_records AS (
93118

94119
ghost AS (
95120
{{ automate_dv.create_ghost_record(src_pk=src_pk, src_hashdiff=src_hashdiff,
96-
src_payload=src_payload, src_extra_columns=src_extra_columns,
97-
src_eff=src_eff, src_ldts=src_ldts,
98-
src_source=src_source, source_model=source_model) }}
121+
src_payload=src_payload, src_extra_columns=src_extra_columns,
122+
src_eff=src_eff, src_ldts=src_ldts,
123+
src_source=src_source, source_model=source_model) }}
99124
),
100125

101126
{%- endif %}
@@ -113,7 +138,7 @@ records_to_insert AS (
113138
SELECT {{ automate_dv.alias_all(source_cols, 'frin') }}
114139
FROM first_record_in_set AS frin
115140
{%- if automate_dv.is_any_incremental() %}
116-
LEFT JOIN LATEST_RECORDS lr
141+
LEFT JOIN latest_records lr
117142
ON {{ automate_dv.multikey(src_pk, prefix=['lr','frin'], condition='=') }}
118143
AND {{ automate_dv.prefix([src_hashdiff], 'lr', alias_target='target') }} = {{ automate_dv.prefix([src_hashdiff], 'frin') }}
119144
WHERE {{ automate_dv.prefix([src_hashdiff], 'lr', alias_target='target') }} IS NULL

0 commit comments

Comments
 (0)