Skip to content

Commit d15aeab

Browse files
author
Alex Higgs
committed
Merge branch 'develop'
2 parents ad08bca + 695ec99 commit d15aeab

10 files changed

+76
-16
lines changed

‎dbt_project.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: 'dbtvault'
2-
version: '0.7.3'
2+
version: '0.7.4'
33
require-dbt-version: [">=0.18.0", "<0.20.0"]
44
config-version: 2
55

‎macros/internal/check_required_parameters.sql

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{%- macro check_required_parameters() -%}
2+
3+
{%- set ns = namespace(missing_parameters=[]) -%}
4+
5+
{%- if kwargs is not none -%}
6+
7+
{%- for k, v in kwargs.items() %}
8+
{%- do ns.missing_parameters.append(k) if v is none -%}
9+
{%- endfor -%}
10+
11+
{%- if ns.missing_parameters -%}
12+
{{- exceptions.raise_compiler_error("Required parameter(s) missing or none in '{}': {}".format(this, ns.missing_parameters | join(", "))) -}}
13+
{%- endif -%}
14+
{%- endif -%}
15+
16+
{%- endmacro -%}

‎macros/internal/expand_column_list.sql

+4-4
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
{%- if not columns -%}
44
{%- if execute -%}
5-
{{ exceptions.raise_compiler_error("Expected a list of columns, got: " ~ columns) }}
5+
{{- exceptions.raise_compiler_error("Expected a list of columns, got: " ~ columns) -}}
66
{%- endif -%}
77
{%- endif -%}
88

@@ -31,7 +31,7 @@
3131
{%- else -%}
3232

3333
{%- if execute -%}
34-
{{ exceptions.raise_compiler_error("Invalid columns object provided. Must be a list of lists, dictionaries or strings.") }}
34+
{{- exceptions.raise_compiler_error("Invalid columns object provided. Must be a list of lists, dictionaries or strings.") -}}
3535
{%- endif %}
3636

3737
{%- endif -%}
@@ -40,11 +40,11 @@
4040
{%- else -%}
4141

4242
{%- if execute -%}
43-
{{ exceptions.raise_compiler_error("Invalid columns object provided. Must be a list.") }}
43+
{{- exceptions.raise_compiler_error("Invalid columns object provided. Must be a list.") -}}
4444
{%- endif %}
4545

4646
{%- endif -%}
4747

48-
{% do return(col_list) %}
48+
{%- do return(col_list) -%}
4949

5050
{%- endmacro -%}

‎macros/internal/multikey.sql

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,13 @@
1616

1717
{%- if condition in ['<>', '!=', '='] -%}
1818
{%- for col in columns -%}
19-
{{ prefix[0] ~ '.' if prefix }}{{ col }} {{ condition }} {{ prefix[1] ~ '.' if prefix }}{{ col }}
19+
{{ (prefix[0] ~ '.') if prefix }}{{ col }} {{ condition }} {{ (prefix[1] ~ '.') if prefix }}{{ col }}
2020
{%- if not loop.last %} {{ operator }} {% endif %}
2121
{% endfor -%}
2222
{%- else -%}
2323
{%- if dbtvault.is_list(columns) -%}
2424
{%- for col in columns -%}
25-
{{ prefix[0] ~ '.' if prefix }}{{ col }} {{ condition if condition else '' }}
25+
{{ (prefix[0] ~ '.') if prefix }}{{ col }} {{ condition if condition else '' }}
2626
{%- if not loop.last -%} {{ "\n " ~ operator }} {% endif -%}
2727
{%- endfor -%}
2828
{%- else -%}

‎macros/staging/stage.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
{% if (source_model is none) and execute %}
1919

2020
{%- set error_message -%}
21-
"Staging error: Missing source_model configuration. A source model name must be provided.
21+
Staging error: Missing source_model configuration. A source model name must be provided.
2222
e.g.
2323
[REF STYLE]
2424
source_model: model_name

‎macros/tables/eff_sat.sql

+10-1
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,14 @@
88

99
{%- macro default__eff_sat(src_pk, src_dfk, src_sfk, src_start_date, src_end_date, src_eff, src_ldts, src_source, source_model) -%}
1010

11+
{{- dbtvault.check_required_parameters(src_pk=src_pk, src_dfk=src_dfk, src_sfk=src_sfk,
12+
src_start_date=src_start_date, src_end_date=src_end_date,
13+
src_eff=src_eff, src_ldts=src_ldts, src_source=src_source,
14+
source_model=source_model) -}}
15+
1116
{%- set source_cols = dbtvault.expand_column_list(columns=[src_pk, src_dfk, src_sfk, src_start_date, src_end_date, src_eff, src_ldts, src_source]) -%}
1217
{%- set fk_cols = dbtvault.expand_column_list(columns=[src_dfk, src_sfk]) -%}
18+
{%- set dfk_cols = dbtvault.expand_column_list(columns=[src_dfk]) -%}
1319
{%- set is_auto_end_dating = config.get('is_auto_end_dating', default=false) %}
1420

1521
{{- dbtvault.prepend_generated_by() }}
@@ -43,7 +49,10 @@ latest_open_eff AS
4349
(
4450
SELECT {{ dbtvault.alias_all(source_cols, 'b') }},
4551
ROW_NUMBER() OVER (
46-
PARTITION BY b.{{ src_pk }}
52+
PARTITION BY
53+
{%- for driving_key in dfk_cols %}
54+
{{ driving_key }}{{ ", " if not loop.last }}
55+
{%- endfor %}
4756
ORDER BY b.{{ src_ldts }} DESC
4857
) AS row_number
4958
FROM {{ this }} AS b

‎macros/tables/hub.sql

+7-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@
88

99
{%- macro default__hub(src_pk, src_nk, src_ldts, src_source, source_model) -%}
1010

11+
{{- dbtvault.check_required_parameters(src_pk=src_pk, src_nk=src_nk,
12+
src_ldts=src_ldts, src_source=src_source,
13+
source_model=source_model) -}}
14+
1115
{%- set source_cols = dbtvault.expand_column_list(columns=[src_pk, src_nk, src_ldts, src_source]) -%}
1216

1317
{%- if model.config.materialized == 'vault_insert_by_rank' %}
@@ -36,9 +40,10 @@ row_rank_{{ source_number }} AS (
3640
{%- endif %}
3741
ROW_NUMBER() OVER(
3842
PARTITION BY {{ src_pk }}
39-
ORDER BY {{ src_ldts }} ASC
43+
ORDER BY {{ src_ldts }}
4044
) AS row_number
4145
FROM {{ ref(src) }}
46+
WHERE {{ dbtvault.multikey(src_pk, condition='IS NOT NULL') }}
4247
QUALIFY row_number = 1
4348
{%- set ns.last_cte = "row_rank_{}".format(source_number) %}
4449
),{{ "\n" if not loop.last }}
@@ -78,7 +83,7 @@ row_rank_union AS (
7883
ORDER BY {{ src_ldts }}, {{ src_source }} ASC
7984
) AS row_rank_number
8085
FROM {{ ns.last_cte }}
81-
WHERE {{ src_pk }} IS NOT NULL
86+
WHERE {{ dbtvault.multikey(src_pk, condition='IS NOT NULL') }}
8287
QUALIFY row_rank_number = 1
8388
{%- set ns.last_cte = "row_rank_union" %}
8489
),

‎macros/tables/link.sql

+10-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@
88

99
{%- macro default__link(src_pk, src_fk, src_ldts, src_source, source_model) -%}
1010

11+
{{- dbtvault.check_required_parameters(src_pk=src_pk, src_fk=src_fk,
12+
src_ldts=src_ldts, src_source=src_source,
13+
source_model=source_model) -}}
14+
1115
{%- set source_cols = dbtvault.expand_column_list(columns=[src_pk, src_fk, src_ldts, src_source]) -%}
1216
{%- set fk_cols = dbtvault.expand_column_list([src_fk]) -%}
1317

@@ -40,6 +44,10 @@ row_rank_{{ source_number }} AS (
4044
ORDER BY {{ src_ldts }} ASC
4145
) AS row_number
4246
FROM {{ ref(src) }}
47+
{%- if source_model | length == 1 %}
48+
WHERE {{ dbtvault.multikey(src_pk, condition='IS NOT NULL') }}
49+
AND {{ dbtvault.multikey(fk_cols, condition='IS NOT NULL') }}
50+
{%- endif %}
4351
QUALIFY row_number = 1
4452
{%- set ns.last_cte = "row_rank_{}".format(source_number) %}
4553
),{{ "\n" if not loop.last }}
@@ -79,7 +87,8 @@ row_rank_union AS (
7987
ORDER BY {{ src_ldts }}, {{ src_source }} ASC
8088
) AS row_rank_number
8189
FROM {{ ns.last_cte }}
82-
WHERE {{ dbtvault.multikey(fk_cols, condition='IS NOT NULL') }}
90+
WHERE {{ src_pk }} IS NOT NULL
91+
AND {{ dbtvault.multikey(fk_cols, condition='IS NOT NULL') }}
8392
QUALIFY row_rank_number = 1
8493
{%- set ns.last_cte = "row_rank_union" %}
8594
),

‎macros/tables/sat.sql

+12-2
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,13 @@
88

99
{%- macro default__sat(src_pk, src_hashdiff, src_payload, src_eff, src_ldts, src_source, source_model) -%}
1010

11+
{{- dbtvault.check_required_parameters(src_pk=src_pk, src_hashdiff=src_hashdiff, src_payload=src_payload,
12+
src_eff=src_eff, src_ldts=src_ldts, src_source=src_source,
13+
source_model=source_model) -}}
14+
1115
{%- set source_cols = dbtvault.expand_column_list(columns=[src_pk, src_hashdiff, src_payload, src_eff, src_ldts, src_source]) -%}
1216
{%- set rank_cols = dbtvault.expand_column_list(columns=[src_pk, src_hashdiff, src_ldts]) -%}
17+
{%- set pk_cols = dbtvault.expand_column_list(columns=[src_pk]) -%}
1318

1419
{%- if model.config.materialized == 'vault_insert_by_rank' %}
1520
{%- set source_cols_with_rank = source_cols + [config.get('rank_column')] -%}
@@ -26,6 +31,9 @@ WITH source_data AS (
2631
FROM {{ ref(source_model) }} AS a
2732
{%- if model.config.materialized == 'vault_insert_by_period' %}
2833
WHERE __PERIOD_FILTER__
34+
AND {{ dbtvault.multikey(src_pk, condition='IS NOT NULL') }}
35+
{% elif model.config.materialized != 'vault_insert_by_rank' and model.config.materialized != 'vault_insert_by_period' %}
36+
WHERE {{ dbtvault.multikey(src_pk, condition='IS NOT NULL') }}
2937
{% endif %}
3038
{%- set source_cte = "source_data" %}
3139
),
@@ -34,6 +42,7 @@ WITH source_data AS (
3442
rank_col AS (
3543
SELECT * FROM source_data
3644
WHERE __RANK_FILTER__
45+
AND {{ dbtvault.multikey(src_pk, condition='IS NOT NULL') }}
3746
{%- set source_cte = "rank_col" %}
3847
),
3948
{% endif -%}
@@ -63,8 +72,9 @@ records_to_insert AS (
6372
FROM {{ source_cte }} AS e
6473
{%- if dbtvault.is_vault_insert_by_period() or dbtvault.is_vault_insert_by_rank() or is_incremental() %}
6574
LEFT JOIN latest_records
66-
ON {{ dbtvault.prefix([src_hashdiff], 'latest_records', alias_target='target') }} = {{ dbtvault.prefix([src_hashdiff], 'e') }}
67-
WHERE {{ dbtvault.prefix([src_hashdiff], 'latest_records', alias_target='target') }} IS NULL
75+
ON {{ dbtvault.prefix([src_pk], 'latest_records', alias_target='target') }} = {{ dbtvault.prefix([src_pk], 'e') }}
76+
WHERE {{ dbtvault.prefix([src_hashdiff], 'latest_records', alias_target='target') }} != {{ dbtvault.prefix([src_hashdiff], 'e') }}
77+
OR {{ dbtvault.prefix([src_hashdiff], 'latest_records', alias_target='target') }} IS NULL
6878
{%- endif %}
6979
)
7080

‎macros/tables/t_link.sql

+13-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,12 @@
88

99
{%- macro default__t_link(src_pk, src_fk, src_payload, src_eff, src_ldts, src_source, source_model) -%}
1010

11+
{{- dbtvault.check_required_parameters(src_pk=src_pk, src_fk=src_fk, src_eff=src_eff,
12+
src_ldts=src_ldts, src_source=src_source,
13+
source_model=source_model) -}}
14+
1115
{%- set source_cols = dbtvault.expand_column_list(columns=[src_pk, src_fk, src_payload, src_eff, src_ldts, src_source]) -%}
16+
{%- set fk_cols = dbtvault.expand_column_list([src_fk]) -%}
1217

1318
{{ dbtvault.prepend_generated_by() }}
1419

@@ -17,9 +22,15 @@ WITH stage AS (
1722
FROM {{ ref(source_model) }}
1823
{%- if model.config.materialized == 'vault_insert_by_period' %}
1924
WHERE __PERIOD_FILTER__
20-
{%- endif %}
21-
{%- if model.config.materialized == 'vault_insert_by_rank' %}
25+
AND {{ dbtvault.multikey(src_pk, condition='IS NOT NULL') }}
26+
AND {{ dbtvault.multikey(fk_cols, condition='IS NOT NULL') }}
27+
{%- elif model.config.materialized == 'vault_insert_by_rank' %}
2228
WHERE __RANK_FILTER__
29+
AND {{ dbtvault.multikey(src_pk, condition='IS NOT NULL') }}
30+
AND {{ dbtvault.multikey(fk_cols, condition='IS NOT NULL') }}
31+
{%- else %}
32+
WHERE {{ dbtvault.multikey(src_pk, condition='IS NOT NULL') }}
33+
AND {{ dbtvault.multikey(fk_cols, condition='IS NOT NULL') }}
2334
{%- endif %}
2435
),
2536
records_to_insert AS (

0 commit comments

Comments
 (0)