Add dbt date shims (pt 2) (#40)

dataders · web-flow · commit d1c741e06310 · 2021-04-29T07:51:51.000-07:00
* needed for dbt_expectations_integration_tests.data_test * enable testing for dbt_expectations * typos * dbt_expectations_integration_test.timeseries_data * ln not supported in TSQL * dbt-utils via dbt-date * add CI * group by explicit column name * TEMP point to fork * test for dbt-expectations * typo * always spin down after * testing * actual logs * repo name not cloned * top-level * per calogica/dbt-expectations#53 * specify outputs * redundant * re-enable * TEMP: disable tests * typo * add dbt-date as submodule * typo * correct directory * for integration tests * dbt-date integration test project * mimic dbt-date ordering * synapse can't handle dateadd hour w/ date type * update upstream * trim ws * only synapse is incompatible * faster CI * unneeded * add deps and required macros * auto-testing * dispatch to synapse * also test synapse * Update .gitmodules * break out into standalone file * new changes * redundant * final two tests * generalize & refresh docs * both packages
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -70,6 +70,28 @@ jobs:
       - store_artifacts:
           path: /root/project/integration_tests/dbt_expectations/target
 
+  integration-dbt-date-azuresql:
+    docker:
+      - image: dbtmsft/msodbc_py:0.5
+    steps:
+      - checkout
+      - azure-cli/install
+      - azure-cli/login-with-service-principal: *azure-creds
+      - run: *pull-submodules
+      - run: *setup-dbt
+      - run:
+          name: "Run Tests - dbt-date"
+          command: |
+            . venv/bin/activate
+            cd integration_tests/dbt_date
+            dbt deps --target azuresql
+            dbt run --target azuresql --full-refresh
+            dbt test --target azuresql
+      - store_artifacts:
+          path: /root/project/integration_tests/dbt_date/logs/dbt.log
+      - store_artifacts:
+          path: /root/project/integration_tests/dbt_date/target
+
   integration-dbt-utils-synapse:
     docker:
       - image: dbtmsft/msodbc_py:0.5
@@ -123,13 +145,37 @@ jobs:
       - store_artifacts:
           path: ./logs
 
+  integration-dbt-date-synapse:
+    docker:
+      - image: dbtmsft/msodbc_py:0.5
+    steps:
+      - checkout
+      - run: *pull-submodules
+      - azure-cli/install
+      - azure-cli/login-with-service-principal: *azure-creds
+      - run: *setup-dbt
+      - run: *start-synapse
+      - run:
+          name: "Run Tests - dbt-date"
+          command: |
+            . venv/bin/activate
+            cd integration_tests/dbt_date
+            dbt deps --target synapse
+            dbt run --target synapse --full-refresh
+            dbt test --target synapse
+      - run: *pause-synapse
+      - store_artifacts:
+          path: ./logs
+
 
 workflows:
   version: 2
   test-all:
     jobs:
-      - integration-dbt-expectations-azuresql: &dbt-context
+      - integration-dbt-utils-azuresql: &dbt-context
           context: DBT_SYNAPSE_PROFILE
-      - integration-dbt-utils-azuresql: *dbt-context
       - integration-dbt-utils-synapse: *dbt-context
+      - integration-dbt-expectations-azuresql: *dbt-context
       - integration-dbt-expectations-synapse: *dbt-context
+      - integration-dbt-date-azuresql: *dbt-context
+      - integration-dbt-date-synapse: *dbt-context
diff --git a/.gitmodules b/.gitmodules
@@ -6,3 +6,7 @@
 	path = dbt-expectations
 	url = https://github.com/calogica/dbt-expectations
 	branch = main
+[submodule "dbt-date"]
+	path = dbt-date
+	url = https://github.com/calogica/dbt-date
+	branch = main
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,16 @@
 # Changelog
 
+## `0.6.7`
+
+## New features
+
+- shim dbt-date, currently passing all tests! [#36](https://github.com/dbt-msft/tsql-utils/pull/36)
+- support for `dbt_utils.generate_series()` [#36](https://github.com/dbt-msft/tsql-utils/pull/36)
+- support on synapse for `dbt_utils.dateadd()` and `dbt_utils.datediff()` [#36](https://github.com/dbt-msft/tsql-utils/pull/36)
+
+### Under the hood
+- add CI infrastucture for dbt-date and dbt-expectations [#35](https://github.com/dbt-msft/tsql-utils/pull/35) thanks to [@b-per](https://github.com/b-per) for all the heaving lifting with [calogica/dbt-expectations #29](https://github.com/calogica/dbt-expectations/pull/29), [calogica/dbt-date #14](https://github.com/calogica/dbt-date/pull/14), and [calogica/dbt-date #15](https://github.com/calogica/dbt-date/pull/15)
+- docs refresh
 
 ## `0.6.6`
 
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -0,0 +1,40 @@
+## Contributions
+
+### Help out!
+
+There are a number of macros that have yet to be implemented for T-SQL. Check out
+- the issues page, or
+- [this file](integration_tests/dbt_utils/dbt_project.yml) for an idea of which macros are not currently supported.
+
+### Dev environment set-up
+
+0. clone this repo
+1. open terminal in top-level dir of project
+2. run the following to install dbt-utils as a submodule
+    `git clone --recursive https://github.com/fishtown-analytics/dbt-utils` 
+3. open terminal in `integration_tests/dbt_utils`
+4. configure your `~/.dbt/profiles.yml` to have an `integration_tests` section with a `sqlserver` target set as the default like `integrations_tests/ci/sample.profiles.yml` does
+5. run all the commands listed in the "Run Tests - dbt-utils" step of `.circleci/config.yml` namely:
+    ```bash
+    dbt deps --target sqlserver
+    dbt seed --target sqlserver --full-refresh
+    dbt run --target sqlserver --full-refresh
+    dbt test --target sqlserver
+    ```
+
+### Multi-Adapter Support
+
+t-sql-utils applies for 2 adapters, sqlserver and synapse.
+
+Therefore, for the time being, a macro should be implemented twice, once for the functionality and once as a reference for the second adapter. 
+
+Imagine an adapter plugin, dbt-synapse, that inherits from dbt-sqlserver.
+For the time being, we need to explicitly reimplement sqlserver__ macros as synapse__ macros. This looks like: 
+
+```
+{% macro synapse__get_tables_by_pattern_sql(field) %}
+    {% do return( tsql_utils.sqlserver__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database)) %}
+{% endmacro %}
+```
+
+TODO: We can make a small change to dbt-core (https://github.com/fishtown-analytics/dbt/issues/2923) that will make the inheritance of dispatched macros work just like the inheritance of other adapter objects, and render the following code redundant.
diff --git a/README.md b/README.md
@@ -5,74 +5,38 @@ that:
 - can be (re)used across dbt projects running on Azure databases
 - define implementations of [dispatched macros](https://docs.getdbt.com/reference/dbt-jinja-functions/adapter/#dispatch) from other packages that can be used on a database that speaks T-SQL: SQL Server, Azure SQL, Azure Synapse, etc.
 
+## Compatibility
+
+This package provides "shims" for:
+- [dbt-utils](https://github.com/fishtown-analytics/dbt-utils) (partial)
+- [dbt-date](https://github.com/calogica/dbt-date) (partial)
+- (coming soon!) [dbt-expectations](https://github.com/calogica/dbt-expectations) (coming soon!)
+
+
+## Usage
+
+Wherever a custom tsql macro exists, dbt_utils adapter dispatch will pass to tsq_utils. This means you can just do `{{dbt_utils.hash('mycolumnname')}}` just like your friends with Snowflake. 
+
+
 ## Installation Instructions
 
 To make use of these TSQL adaptations in your dbt project, you must do two things:
-1. Install both `dbt-utils` and `tsql-utils` by adding the following to your `packages.yml`
+1. Install both and `tsql-utils` and any of the compatible packages listed above by them to your `packages.yml`
     ```yaml
     packages:
-      - package: fishtown-analytics/dbt_utils
-        version: 0.6.4.1
+      # and/or calogica/dbt-date; calogica/dbt-expectations
+      - package: fishtown-analytics/dbt_utils 
+        version: {SEE DBT HUB FOR NEWEST VERSION}
       - package: dbt-msft/tsql_utils
-        version: 0.6.4.1
+        version: {SEE DBT HUB FOR NEWEST VERSION}
     ```
-2. Tell `dbt-utils` to also look for the `tsql-utils` macros by adding this section to your `dbt_project.yml`
+2. Tell the supported package to also look for the `tsql-utils` macros by adding the relevant variables to your `dbt_project.yml`
     ```yaml
     vars:
       dbt_utils_dispatch_list: ['tsql_utils']
+      dbt_date_dispatch_list: ['tsql_utils']
+      dbt_expectations_dispatch_list: ['tsql_utils']
     ```
 Check [dbt Hub](https://hub.getdbt.com) for the latest installation 
 instructions, or [read the docs](https://docs.getdbt.com/docs/package-management) 
-for more information on installing packages.
-
-----
-
-## Usage
-
-Wherever a custom tsql macro exists, dbt_utils adapter dispatch will pass to tsq_utils. This means you can just do `{{dbt_utils.hash('mycolumnname')}}` just like your friends with Snowflake. 
-
-## Compatibility
-
-This package provides "shims" for:
-- [dbt_utils](https://github.com/fishtown-analytics/dbt-utils) (partial)
-
-## Contributions
-
-### Help out!
-
-There are a number of macros that have yet to be implemented for T-SQL. Check out
-- the issues page, or
-- [this file](integration_tests/dbt_utils/dbt_project.yml) for an idea of which macros are not currently supported.
-
-### Dev environment set-up
-
-0. clone this repo
-1. open terminal in top-level dir of project
-2. run the following to install dbt-utils as a submodule
-    `git clone --recursive https://github.com/fishtown-analytics/dbt-utils` 
-3. open terminal in `integration_tests/dbt_utils`
-4. configure your `~/.dbt/profiles.yml` to have an `integration_tests` section with a `sqlserver` target set as the default like `integrations_tests/ci/sample.profiles.yml` does
-5. run all the commands listed in the "Run Tests - dbt-utils" step of `.circleci/config.yml` namely:
-    ```bash
-    dbt deps --target sqlserver
-    dbt seed --target sqlserver --full-refresh
-    dbt run --target sqlserver --full-refresh
-    dbt test --target sqlserver
-    ```
-
-### Multi-Adapter Support
-
-t-sql-utils applies for 2 adapters, sqlserver and synapse.
-
-Therefore, for the time being, a macro should be implemented twice, once for the functionality and once as a reference for the second adapter. 
-
-Imagine an adapter plugin, dbt-synapse, that inherits from dbt-sqlserver.
-For the time being, we need to explicitly reimplement sqlserver__ macros as synapse__ macros. This looks like: 
-
-```
-{% macro synapse__get_tables_by_pattern_sql(field) %}
-    {% do return( tsql_utils.sqlserver__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database)) %}
-{% endmacro %}
-```
-
-TODO: We can make a small change to dbt-core (https://github.com/fishtown-analytics/dbt/issues/2923) that will make the inheritance of dispatched macros work just like the inheritance of other adapter objects, and render the following code redundant.
+for more information on installing packages.
diff --git a/dbt-date b/dbt-date
@@ -0,0 +1 @@
+Subproject commit e58387dd6e85b116e693e2675193d71547a09a35
diff --git a/integration_tests/dbt_date/dbt_project.yml b/integration_tests/dbt_date/dbt_project.yml
@@ -0,0 +1,33 @@
+name: 'tsql_utils_dbt_date_integration_tests'
+version: '1.0'
+config-version: 2
+
+profile: 'integration_tests'
+
+source-paths: ["models"]
+analysis-paths: ["analysis"]
+test-paths: ["tests"]
+data-paths: ["data"]
+macro-paths: ["macros"]
+
+target-path: "target"
+clean-targets:
+    - "target"
+    - "dbt_modules"
+
+require-dbt-version: ">=0.18.1"
+
+vars:
+  'dbt_date:time_zone': 'Pacific Standard Time'
+  dbt_utils_dispatch_list: ['tsql_utils']
+  dbt_date_dispatch_list: ['tsql_utils']
+
+quoting:
+    database: true
+    identifier: true
+    schema: true
+
+models:
+  dbt_date_integration_tests:
+    +schema: dbt_date_integration_tests
+    +materialized: table
diff --git a/integration_tests/dbt_date/packages.yml b/integration_tests/dbt_date/packages.yml
@@ -0,0 +1,5 @@
+
+packages:
+  - local: ../../ # tsql_utils main macros
+  - local: ../../dbt-date # actual dbt-expectations package as submodule
+  - local: ../../dbt-date/integration_tests # nothing here for now
diff --git a/integration_tests/dbt_expectations/dbt_project.yml b/integration_tests/dbt_expectations/dbt_project.yml
@@ -17,10 +17,10 @@ clean-targets:         # directories to be removed by `dbt clean`
     - "dbt_modules"
 
 vars:
+  'dbt_date:time_zone': 'Pacific Standard Time'
   dbt_utils_dispatch_list: ['tsql_utils']
-  dbt_expectations_dispatch_list: ['tsql_utils']
   dbt_date_dispatch_list: ['tsql_utils']
-  'dbt_date:time_zone': 'Pacific Standard Time'
+  dbt_expectations_dispatch_list: ['tsql_utils']
 
 models:
   dbt_expectations_integration_tests:
diff --git a/macros/dbt_date/calendar_date/convert_timezones.sql b/macros/dbt_date/calendar_date/convert_timezones.sql
diff --git a/macros/dbt_date/calendar_date/date_part.sql b/macros/dbt_date/calendar_date/date_part.sql
@@ -0,0 +1,7 @@
+{% macro sqlserver__date_part(datepart, date) -%}
+    datepart({{ datepart }}, {{ date }})
+{%- endmacro %}
+
+{% macro synapse__date_part(datepart, date) -%}
+    {% do return( tsql_utils.sqlserver__date_part(datepart, date)) %}
+{%- endmacro -%}
diff --git a/macros/dbt_date/calendar_date/day_name.sql b/macros/dbt_date/calendar_date/day_name.sql
@@ -0,0 +1,8 @@
+{%- macro sqlserver__day_name(date, short) -%}
+{%- set f = 'ddd' if short else 'dddd' -%}
+    format({{ date }}, '{{ f }}')
+{%- endmacro %}
+
+{% macro synapse__day_name(date, short) -%}
+    {% do return( tsql_utils.sqlserver__day_name(date, short)) %}
+{%- endmacro -%}
diff --git a/macros/dbt_date/calendar_date/day_of_week.sql b/macros/dbt_date/calendar_date/day_of_week.sql
@@ -0,0 +1,20 @@
+
+{%- macro sqlserver__day_of_week(date, isoweek) -%}
+
+    {%- set dow = dbt_date.date_part('weekday', date) -%}
+
+    {%- if isoweek -%}
+    case
+        -- Shift start of week from Sunday (1) to Monday (2)
+        when {{ dow }} = 1 then 7
+        else {{ dow }} - 1
+    end
+    {%- else -%}
+    {{ dow }}
+    {%- endif -%}
+
+{%- endmacro %}
+
+{% macro synapse__day_of_week(date, isoweek) -%}
+    {% do return( tsql_utils.sqlserver__day_of_week(date, isoweek)) %}
+{%- endmacro -%}
diff --git a/macros/dbt_date/calendar_date/iso_week_of_year.sql b/macros/dbt_date/calendar_date/iso_week_of_year.sql
@@ -0,0 +1,7 @@
+{%- macro sqlserver__iso_week_of_year(date) -%}
+cast({{ dbt_date.date_part('iso_week', date) }} as {{ dbt_utils.type_int() }}) 
+{%- endmacro %}
+
+{% macro synapse__iso_week_of_year(date) -%}
+    {% do return( tsql_utils.sqlserver__iso_week_of_year(date)) %}
+{%- endmacro -%}
diff --git a/macros/dbt_date/calendar_date/iso_week_start.sql b/macros/dbt_date/calendar_date/iso_week_start.sql
@@ -0,0 +1,7 @@
+{%- macro sqlserver__iso_week_start(date) -%}
+    cast(dateadd(week, datediff(week, 0, dateadd(day, -1, {{date}})), 0) as date)
+{%- endmacro %}
+
+{% macro synapse__iso_week_start(date) -%}
+    {% do return( tsql_utils.sqlserver__iso_week_start(date)) %}
+{%- endmacro -%}
diff --git a/macros/dbt_date/calendar_date/month_name.sql b/macros/dbt_date/calendar_date/month_name.sql
@@ -0,0 +1,8 @@
+{%- macro sqlserver__month_name(date, short) -%}
+{%- set f = 'MMM' if short else 'MMMM' -%}
+    format({{ date }}, '{{ f }}')
+{%- endmacro %}
+
+{% macro synapse__month_name(date, short) -%}
+    {% do return( tsql_utils.sqlserver__month_name(date, short)) %}
+{%- endmacro -%}
diff --git a/macros/dbt_date/calendar_date/week_end.sql b/macros/dbt_date/calendar_date/week_end.sql
@@ -0,0 +1,8 @@
+{%- macro sqlserver__week_end(date) -%}
+{%- set dt = dbt_date.week_start(date) -%}
+{{ dbt_date.n_days_away(6, dt) }}
+{%- endmacro %}
+
+{% macro synapse__week_end(date) -%}
+    {% do return( tsql_utils.sqlserver__week_end(date)) %}
+{%- endmacro -%}
diff --git a/macros/dbt_date/calendar_date/week_of_year.sql b/macros/dbt_date/calendar_date/week_of_year.sql
@@ -0,0 +1,7 @@
+{%- macro sqlserver__week_of_year(date) -%}
+cast({{ dbt_date.date_part('week', dbt_utils.dateadd('day',-1,date)) }} as {{ dbt_utils.type_int() }})
+{%- endmacro %}
+
+{% macro synapse__week_of_year(date) -%}
+    {% do return( tsql_utils.sqlserver__week_of_year(date)) %}
+{%- endmacro -%}
diff --git a/macros/dbt_date/calendar_date/week_start.sql b/macros/dbt_date/calendar_date/week_start.sql
@@ -0,0 +1,8 @@
+{%- macro sqlserver__week_start(date) -%}
+-- Sunday as week start date
+cast({{ dbt_utils.dateadd('day', -1, dbt_utils.date_trunc('week', dbt_utils.dateadd('day', 1, date))) }} as date)
+{%- endmacro %}
+
+{% macro synapse__week_start(date) -%}
+    {% do return( tsql_utils.sqlserver__week_start(date)) %}
+{%- endmacro -%}
