Skip to content

Commit bd67ccc

Browse files
author
Alex Higgs
committed
Merge branch 'release/0.7.6-b1'
2 parents a2e5780 + 0cb4a0d commit bd67ccc

10 files changed

+711
-5
lines changed

‎.circleci/config.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ jobs:
5656
integration:
5757
docker:
5858
- image: cimg/python:3.8.5
59-
parallelism: 15
59+
parallelism: 20
6060
steps:
6161
- build_test_env
6262
- run:

‎macros/materialisations/incremental_bridge_helpers.sql

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{% macro is_bridge_incremental() %}
2+
{#-- do not run introspective queries in parsing #}
3+
{% if not execute %}
4+
{{ return(False) }}
5+
{% else %}
6+
{% set relation = adapter.get_relation(this.database, this.schema, this.table) %}
7+
8+
{{ return(relation is not none
9+
and relation.type == 'table'
10+
and model.config.materialized == 'bridge_incremental'
11+
and not flags.FULL_REFRESH) }}
12+
{% endif %}
13+
{% endmacro %}
14+
15+
{% macro incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %}
16+
{%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
17+
{%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
18+
19+
TRUNCATE TABLE {{ target_relation }};
20+
21+
INSERT INTO {{ target_relation }} ({{ dest_cols_csv }})
22+
(
23+
SELECT {{ dest_cols_csv }}
24+
FROM {{ tmp_relation }}
25+
);
26+
{%- endmacro %}

‎macros/materialisations/incremental_bridge_materialization.sql

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{% materialization bridge_incremental, default -%}
2+
3+
{% set full_refresh_mode = flags.FULL_REFRESH %}
4+
5+
{% set target_relation = this %}
6+
{% set existing_relation = load_relation(this) %}
7+
{% set tmp_relation = make_temp_relation(this) %}
8+
9+
{{ run_hooks(pre_hooks, inside_transaction=False) }}
10+
11+
-- `BEGIN` happens here:
12+
{{ run_hooks(pre_hooks, inside_transaction=True) }}
13+
14+
{% set to_drop = [] %}
15+
{% if existing_relation is none %}
16+
{% set build_sql = create_table_as(False, target_relation, sql) %}
17+
{% elif existing_relation.is_view or full_refresh_mode %}
18+
{#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
19+
{% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
20+
{% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
21+
{% do adapter.drop_relation(backup_relation) %}
22+
23+
{% do adapter.rename_relation(target_relation, backup_relation) %}
24+
{% set build_sql = create_table_as(False, target_relation, sql) %}
25+
{% do to_drop.append(backup_relation) %}
26+
{% else %}
27+
28+
{% set tmp_relation = make_temp_relation(target_relation) %}
29+
{% do run_query(create_table_as(True, tmp_relation, sql)) %}
30+
{% do adapter.expand_target_column_types(
31+
from_relation=tmp_relation,
32+
to_relation=target_relation) %}
33+
{% set build_sql = dbtvault.incremental_bridge_replace(tmp_relation, target_relation) %}
34+
{% endif %}
35+
36+
{% call statement("main") %}
37+
{{ build_sql }}
38+
{% endcall %}
39+
40+
{{ run_hooks(post_hooks, inside_transaction=True) }}
41+
42+
-- `COMMIT` happens here
43+
{% do adapter.commit() %}
44+
45+
{% for rel in to_drop %}
46+
{% do adapter.drop_relation(rel) %}
47+
{% endfor %}
48+
49+
{{ run_hooks(post_hooks, inside_transaction=False) }}
50+
51+
{{ return({'relations': [target_relation]}) }}
52+
53+
{%- endmaterialization %}

‎macros/materialisations/incremental_pit_helper.sql

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{% macro is_pit_incremental() %}
2+
{#-- do not run introspective queries in parsing #}
3+
{% if not execute %}
4+
{{ return(False) }}
5+
{% else %}
6+
{% set relation = adapter.get_relation(this.database, this.schema, this.table) %}
7+
8+
{{ return(relation is not none
9+
and relation.type == 'table'
10+
and model.config.materialized == 'pit_incremental'
11+
and not flags.FULL_REFRESH) }}
12+
{% endif %}
13+
{% endmacro %}
14+
15+
{% macro incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %}
16+
{%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
17+
{%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
18+
19+
TRUNCATE TABLE {{ target_relation }};
20+
21+
INSERT INTO {{ target_relation }} ({{ dest_cols_csv }})
22+
(
23+
SELECT {{ dest_cols_csv }}
24+
FROM {{ tmp_relation }}
25+
);
26+
{%- endmacro %}

‎macros/materialisations/incremental_pit_materialization.sql

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{% materialization pit_incremental, default -%}
2+
3+
{% set full_refresh_mode = flags.FULL_REFRESH %}
4+
5+
{% set target_relation = this %}
6+
{% set existing_relation = load_relation(this) %}
7+
{% set tmp_relation = make_temp_relation(this) %}
8+
9+
{{ run_hooks(pre_hooks, inside_transaction=False) }}
10+
11+
-- `BEGIN` happens here:
12+
{{ run_hooks(pre_hooks, inside_transaction=True) }}
13+
14+
{% set to_drop = [] %}
15+
{% if existing_relation is none %}
16+
{% set build_sql = create_table_as(False, target_relation, sql) %}
17+
{% elif existing_relation.is_view or full_refresh_mode %}
18+
{#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
19+
{% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
20+
{% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
21+
{% do adapter.drop_relation(backup_relation) %}
22+
23+
{% do adapter.rename_relation(target_relation, backup_relation) %}
24+
{% set build_sql = create_table_as(False, target_relation, sql) %}
25+
{% do to_drop.append(backup_relation) %}
26+
{% else %}
27+
28+
{% set tmp_relation = make_temp_relation(target_relation) %}
29+
{% do run_query(create_table_as(True, tmp_relation, sql)) %}
30+
{% do adapter.expand_target_column_types(
31+
from_relation=tmp_relation,
32+
to_relation=target_relation) %}
33+
{% set build_sql = dbtvault.incremental_pit_replace(tmp_relation, target_relation) %}
34+
{% endif %}
35+
36+
{% call statement("main") %}
37+
{{ build_sql }}
38+
{% endcall %}
39+
40+
{{ run_hooks(post_hooks, inside_transaction=True) }}
41+
42+
-- `COMMIT` happens here
43+
{% do adapter.commit() %}
44+
45+
{% for rel in to_drop %}
46+
{% do adapter.drop_relation(rel) %}
47+
{% endfor %}
48+
49+
{{ run_hooks(post_hooks, inside_transaction=False) }}
50+
51+
{{ return({'relations': [target_relation]}) }}
52+
53+
{%- endmaterialization %}

‎macros/materialisations/shared_helpers.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212

1313
{%- macro is_any_incremental() -%}
14-
{%- if dbtvault.is_vault_insert_by_period() or dbtvault.is_vault_insert_by_rank() or is_incremental() -%}
14+
{%- if dbtvault.is_vault_insert_by_period() or dbtvault.is_vault_insert_by_rank() or dbtvault.is_pit_incremental() or dbtvault.is_bridge_incremental() or is_incremental() -%}
1515
{%- do return(true) -%}
1616
{%- else -%}
1717
{%- do return(false) -%}

0 commit comments

Comments
 (0)