.github/workflows/ado_sync.yml

@@ -0,0 +1,32 @@
+name: Sync issue to Azure DevOps work item
+
+on:
+  issues:
+    types:
+      [opened, edited, deleted, closed, reopened, labeled, unlabeled, assigned]
+  issue_comment:
+    types: [created, edited, deleted]
+
+concurrency:
+  group: issue-${{ github.event.issue.number }}
+  cancel-in-progress: false
+
+jobs:
+  alert:
+    if: ${{ !github.event.issue.pull_request }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: danhellem/github-actions-issue-to-work-item@master
+        env:
+          ado_token: "${{ secrets.ADO_PERSONAL_ACCESS_TOKEN }}"
+          github_token: "${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}"
+          ado_organization: "${{ vars.ADO_ORGANIZATION }}"
+          ado_project: "${{ vars.ADO_PROJECT_NAME }}"
+          ado_area_path: "${{ vars.ADO_AREA_PATH }}"
+          ado_iteration_path: "${{ vars.ADO_ITERATION_PATH }}"
+          ado_wit: ${{ (contains(github.event.issue.title, 'FEATURE') || contains(github.event.issue.title, 'DOCS')) && 'Product Backlog Item' || 'Bug' }}         
+          ado_new_state: "New"
+          ado_active_state: ${{ (contains(github.event.issue.title, 'FEATURE') || contains(github.event.issue.title, 'DOCS')) && 'Active' || 'Committed' }}         
+          ado_close_state: ${{ (contains(github.event.issue.title, 'FEATURE') || contains(github.event.issue.title, 'DOCS')) && 'Closed' || 'Done' }}  
+          ado_bypassrules: true
+          log_level: 100

NOTICE

@@ -1,5 +1,5 @@
 AutomateDV (f.k.a dbtvault)
-Copyright 2019-2023 Business Thinking Ltd (Trading as Datavault).
+Copyright 2019-2024 Business Thinking Ltd (Trading as Datavault).
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

README.md

@@ -7,13 +7,13 @@
 </div>
 <div align="center">
 
-  [![dbt Versions](https://img.shields.io/badge/compatible%20dbt%20versions-%3E=1.3%20%3C=1.4.x-orange?logo=dbt)](https://automate-dv.readthedocs.io/en/latest/versions/)
+  [![dbt Versions](https://img.shields.io/badge/compatible%20dbt%20versions-%3E=1.4%20%3C=1.8.x-orange?logo=dbt)](https://automate-dv.readthedocs.io/en/latest/versions/)
 
 </div>
 
 [Changelog and past doc versions](https://automate-dv.readthedocs.io/en/latest/changelog/)
 
-# AutomateDV by [Datavault](https://www.data-vault.co.uk)
+# AutomateDV by [Datavault](https://www.data-vault.com)
 
 Build your own Data Vault data warehouse! AutomateDV is a free to use dbt package that generates & executes the ETL you need to run a Data Vault 2.0 Data Warehouse on your data platform.
 
@@ -32,16 +32,16 @@ Learn quickly with our worked example:
 
 - [Read the docs](https://automate-dv.readthedocs.io/en/latest/worked_example/)
 
-- [Project Repository](https://github.com/Datavault-UK/snowflakeDemo)
+- [Project Repository](https://github.com/Datavault-UK/automate-dv-demo)
 
 ## Supported platforms:
 
-[Platform support matrix](https://automate-dv.readthedocs.io/en/latest/macros/#platform-support)
+[Platform support matrix](https://automate-dv.readthedocs.io/en/latest/platform_support/
 
 ## Installation
 
-Check [dbt Hub](https://hub.getdbt.com/datavault-uk/automate_dv/latest/) for the latest installation instructions, 
-or [read the docs](https://docs.getdbt.com/docs/building-a-dbt-project/package-management/) for more information on installing packages.
+Check [dbt Hub](https://hub.getdbt.com/Datavault-UK/automate_dv/latest/) for the latest installation instructions, 
+or [read the docs](https://docs.getdbt.com/docs/build/packages) for more information on installing packages.
 
 ## Usage
 
@@ -68,6 +68,7 @@ Talk to our developers and other members of our growing community, get support a
 
 [![Join our Slack](https://img.shields.io/badge/Slack-Join-yellow?style=flat&logo=slack)](https://join.slack.com/t/dbtvault/shared_invite/enQtODY5MTY3OTIyMzg2LWJlZDMyNzM4YzAzYjgzYTY0MTMzNTNjN2EyZDRjOTljYjY0NDYyYzEwMTlhODMzNGY3MmU2ODNhYWUxYmM2NjA)
 
+
 ## Social 
 
 [![Twitter Follow](https://img.shields.io/badge/Twitter-1DA1F2?style=for-the-badge&logo=twitter&logoColor=white)](https://twitter.com/Automate_DV)
@@ -76,6 +77,7 @@ Talk to our developers and other members of our growing community, get support a
 
 [![Youtube](https://img.shields.io/badge/YouTube-FF0000?style=for-the-badge&logo=youtube&logoColor=white)](https://www.youtube.com/@AutomateDV)
 
+
 ## Awards
 
 <p align="left">

dbt_project.yml

@@ -1,14 +1,14 @@
 name: automate_dv
-version: 0.10.0
-require-dbt-version: [">=1.0.0", "<2.0.0"]
+version: 0.11.0
+require-dbt-version: [ ">=1.0.0", "<2.0.0" ]
 config-version: 2
 
-model-paths: ["models"]
-analysis-paths: ["analyses"]
-test-paths: ["tests"]
-seed-paths: ["seeds"]
-macro-paths: ["macros"]
-docs-paths: ["docs"]
+model-paths: [ "models" ]
+analysis-paths: [ "analyses" ]
+test-paths: [ "tests" ]
+seed-paths: [ "seeds" ]
+macro-paths: [ "macros" ]
+docs-paths: [ "docs" ]
 
 target-path: "target"
 clean-targets:

macros/internal/helpers/dateadd.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

macros/internal/helpers/is_checks.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

macros/materialisations/error_messages.sql → macros/internal/helpers/logging/error_messages.sql

@@ -1,31 +1,9 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 
 
-{% macro wrap_warning(warning_message) %}
-
-    {%- set new_message = [] -%}
-    {%- set length_list = [] -%}
-
-    {%- for ln in warning_message.split('\n') -%}
-        {%- do new_message.append((ln | trim)) -%}
-        {%- do length_list.append((ln | length)) -%}
-    {%- endfor -%}
-
-    {%- set max_line_length = length_list | max -%}
-    {%- set padding_length = (max_line_length - 7) // 2 -%}
-
-    {%- set border = modules.itertools.repeat('=', padding_length) | join ('') ~ 'WARNING' ~ modules.itertools.repeat('=', padding_length) | join ('') -%}
-
-    {%- set wrapped_message = '\n' ~ border ~ '\n' ~ new_message | join('\n') ~ '\n' ~ border -%}
-
-    {%- do return(wrapped_message) -%}
-
-{% endmacro %}
-
-
 {%- macro datepart_too_small_error(period) -%}
 
     {%- set message -%}
@@ -35,10 +13,7 @@
     Please see: https://automate-dv.readthedocs.io/en/latest/materialisations/
     {%- endset -%}
 
-    {%- if execute -%}
-    {{- exceptions.raise_compiler_error(message) -}}
-    {%- endif -%}
-
+    {%- do automate_dv.log_error(message) -%}
 {%- endmacro -%}
 
 
@@ -51,10 +26,7 @@
     Please see: https://automate-dv.readthedocs.io/en/latest/materialisations/
     {%- endset -%}
 
-    {%- if execute -%}
-    {{- exceptions.warn(automate_dv.wrap_warning(message)) -}}
-    {%- endif -%}
-
+    {%- do automate_dv.log_warning(message) -%}
 {%- endmacro -%}
 
 
@@ -67,10 +39,7 @@
     Please see: https://automate-dv.readthedocs.io/en/latest/materialisations/
     {%- endset %}
 
-    {%- if execute -%}
-    {{- exceptions.raise_compiler_error(message) -}}
-    {%- endif -%}
-
+    {%- do automate_dv.log_warning(message) -%}
 {%- endmacro -%}
 
 
@@ -82,8 +51,46 @@
     Using this functionality in a live production setting may result in unpredictable outcomes, data loss, or system instability.
     {%- endset -%}
 
-    {%- if execute -%}
-    {{- exceptions.warn(automate_dv.wrap_warning(message)) -}}
-    {%- endif -%}
+    {%- do automate_dv.log_warning(message) -%}
+{%- endmacro -%}
+
+
+{%- macro currently_disabled_error(func_name) -%}
+
+    {%- set message -%}
+    This functionality ({{ func_name }}) is currently disabled for dbt-sqlserver 1.7.x,
+    please revert to dbt-sqlserver 1.4.3 and AutomateDV 0.10.1 to use {{ func_name }}.
+
+    This is due to a suspected bug with the SQLServer Adapter in the 1.7.x version.
+    We are actively working to get this fixed. Thank you for your understanding.
+    {%- endset -%}
+
+    {%- do automate_dv.log_error(message) -%}
+{%- endmacro -%}
+
+
+{%- macro materialisation_deprecation_warning() -%}
+
+    {%- set message -%}
+        DEPRECATED: Since AutomateDV v0.11.0, vault_insert_by_x materialisations are now deprecated.
+            These materialisation were initially designed to provide an option for rapid iterative development of
+            incremental loading patterns in local environments for development and testing, allowing users to bypass
+            the need for a comprehensive PSA or delta-loading solution. They are being deprecated to encourage the use
+            of more robust solutions.
+    {%- endset -%}
+
+    {%- do automate_dv.log_warning(message) -%}
+{%- endmacro -%}
+
+
+{%- macro pit_bridge_deprecation_warning() -%}
+
+    {%- set message -%}
+        DEPRECATED: Since AutomateDV v0.11.0, the pit() and bridge() macros are now deprecated.
+            This is because they are not currently fit-for-purpose and need significant usability
+            and peformance improvements, as well as a design overhaul.
+            Improved implementations will be released in a future version of AutomateDV.
+    {%- endset -%}
 
+    {%- do automate_dv.log_warning(message) -%}
 {%- endmacro -%}

macros/internal/helpers/logging/log_relation_sources.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 
@@ -9,7 +9,7 @@
 
 {% macro default__log_relation_sources(relation, source_count) %}
 
-    {%- if 'docs' not in invocation_args_dict['rpc_method'] and execute -%}
+    {%- if execute and automate_dv.is_something(invocation_args_dict.get('which')) and invocation_args_dict.get('which') != 'docs' -%}
 
         {%- do dbt_utils.log_info('Loading {} from {} source(s)'.format("{}.{}.{}".format(relation.database, relation.schema, relation.identifier),
                                                                         source_count)) -%}
@@ -18,7 +18,7 @@
 
 {% macro databricks__log_relation_sources(relation, source_count) %}
 
-    {%- if 'docs' not in invocation_args_dict['rpc_method'] and execute -%}
+    {%- if execute and automate_dv.is_something(invocation_args_dict.get('which')) and invocation_args_dict.get('which') != 'docs' -%}
 
         {%- do dbt_utils.log_info('Loading {} from {} source(s)'.format("{}.{}".format(relation.schema, relation.identifier),
                                                                         source_count)) -%}

macros/internal/helpers/logging/logging_helpers.sql

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) Business Thinking Ltd. 2019-2025
+ * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
+ */
+
+
+{% macro wrap_message(message, type='WARNING') %}
+
+    {%- set new_message = [] -%}
+    {%- set length_list = [] -%}
+
+    {%- for ln in message.split('\n') -%}
+        {%- do new_message.append((ln | trim)) -%}
+        {%- do length_list.append((ln | length)) -%}
+    {%- endfor -%}
+
+    {%- set max_line_length = length_list | max -%}
+    {%- set padding_length = (max_line_length - 7) // 2 -%}
+
+    {%- set border = modules.itertools.repeat('=', padding_length) | join ('') ~ (type | upper) ~ modules.itertools.repeat('=', padding_length) | join ('') -%}
+
+    {%- set wrapped_message = '\n' ~ border ~ '\n' ~ new_message | join('\n') ~ '\n' ~ border -%}
+
+    {%- do return(wrapped_message) -%}
+
+{% endmacro %}
+
+
+{%- macro log_warning(message) -%}
+
+    {%- set message = automate_dv.wrap_message(message) -%}
+
+    {%- if execute and automate_dv.is_something(invocation_args_dict.get('which')) and invocation_args_dict.get('which') != 'compile' -%}
+        {%- do exceptions.warn(message) -%}
+    {%- endif -%}
+
+{%- endmacro -%}
+
+
+{%- macro log_error(message) -%}
+
+    {%- set message = automate_dv.wrap_message(message, type='ERROR') -%}
+
+    {%- if execute and automate_dv.is_something(invocation_args_dict.get('which')) and invocation_args_dict.get('which') != 'compile' -%}
+        {%- do exceptions.raise_compiler_error(message) -%}
+    {%- endif -%}
+
+{%- endmacro -%}

macros/internal/helpers/prepend_generated_by.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

macros/internal/helpers/stage_processing_macros/extract_column_names.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

macros/internal/helpers/stage_processing_macros/extract_null_column_names.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

macros/internal/helpers/stage_processing_macros/print_list.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

macros/internal/helpers/stage_processing_macros/process_columns_to_escape.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

macros/internal/helpers/stage_processing_macros/process_columns_to_select.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

macros/internal/helpers/stage_processing_macros/process_hash_column_excludes.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

macros/internal/helpers/timestamp_add.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

macros/internal/metadata_processing/alias.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

macros/internal/metadata_processing/alias_all.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

macros/internal/metadata_processing/as_constant.sql

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2025
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */