Skip to content

Commit b8eed97

Browse files
datadersdataders
authored
Merge pull request #332 from dbt-labs/reenable-sqlserver-and-synapse-support
Reenable sqlserver and synapse support
2 parents 6528e75 + 705e3eb commit b8eed97

15 files changed

+2147
-31
lines changed

‎.github/workflows/ci.yml

+9
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,18 @@ jobs:
3535
# bigquery
3636
BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
3737
BIGQUERY_SCHEMA: "integration_tests_bigquery_${{ github.run_number }}"
38+
# synapse
39+
SYNAPSE_DRIVER: ${{ vars.SYNAPSE_DRIVER }}
40+
SYNAPSE_HOST: ${{ vars.SYNAPSE_HOST }}
41+
SYNAPSE_PORT: ${{ vars.SYNAPSE_PORT }}
42+
SYNAPSE_DATABASE: ${{ vars.SYNAPSE_DATABASE }}
43+
SYNAPSE_AUTHENTICATION: ${{ vars.SYNAPSE_AUTHENTICATION }}
44+
SYNAPSE_TENANT_ID: ${{ vars.SYNAPSE_TENANT_ID }}
45+
SYNAPSE_CLIENT_ID: ${{ vars.SYNAPSE_CLIENT_ID }}
3846

3947
secrets:
4048
DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }}
4149
SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
4250
DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.DBT_ENV_SECRET_SNOWFLAKE_PASS }}
4351
BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}
52+
SYNAPSE_CLIENT_SECRET: ${{ secrets.SYNAPSE_CLIENT_SECRET }}

‎.python-version

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3.11

‎CHANGELOG.md

+8
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11
# Changelog
22

3+
## dbt-external-tables v0.11.0
4+
5+
### Synapse & SQL Server
6+
* Reenable sqlserver and synapse support https://github.com/dbt-labs/dbt-external-tables/pull/332
7+
8+
9+
**Full Changelog**: https://github.com/dbt-labs/dbt-external-tables/compare/0.10.1...0.10.0
10+
311
## dbt-external-tables v0.10.1
412

513
* [FIX] OOPS! Revert https://github.com/dbt-labs/dbt-external-tables/pull/330 "stage_external_sources Comparing source_name of the node instead of the name of the node" by @dataders in https://github.com/dbt-labs/dbt-external-tables/pull/330

‎integration_tests/macros/plugins/sqlserver/prep_external.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{% macro sqlserver__prep_external() %}
1+
{% macro fabric__prep_external() %}
22

33
{% set external_data_source = target.schema ~ '.dbt_external_tables_testing' %}
44

‎integration_tests/profiles.yml

+8-5
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,14 @@ integration_tests:
4949

5050
synapse:
5151
type: synapse
52-
driver: "ODBC Driver 17 for SQL Server"
53-
port: 1433
54-
host: "{{ env_var('SYNAPSE_TEST_SERVER') }}.sql.azuresynapse.net"
55-
database: "{{ env_var('SYNAPSE_TEST_DBNAME') }}"
56-
authentication: CLI
52+
driver: "{{ env_var('SYNAPSE_DRIVER') }}"
53+
port: "{{ env_var('SYNAPSE_PORT') }}"
54+
host: "{{ env_var('SYNAPSE_HOST') }}"
55+
database: "{{ env_var('SYNAPSE_DATABASE') }}"
56+
authentication: "{{ env_var('SYNAPSE_AUTHENTICATION') }}"
57+
tenant_id: "{{ env_var('SYNAPSE_TENANT_ID') }}"
58+
client_id: "{{ env_var('SYNAPSE_CLIENT_ID') }}"
59+
client_secret: "{{ env_var('SYNAPSE_CLIENT_SECRET') }}"
5760
schema: dbt_external_tables_integration_tests_synapse
5861
threads: 1
5962

‎integration_tests/test.env.sample

+13-16
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,37 @@
11
# gh secret set -f integration_tests/test.env -e ci_testing
22

33
# redshift
4-
REDSHIFT_HOST=
5-
REDSHIFT_USER=
64
DBT_ENV_SECRET_REDSHIFT_PASS=
7-
REDSHIFT_PORT=
8-
REDSHIFT_DBNAME=
9-
REDSHIFT_SCHEMA=
10-
REDSHIFT_SPECTRUM_IAM_ROLE=
5+
# local testing only
6+
# REDSHIFT_SCHEMA=
117

128
# snowflake
139
SNOWFLAKE_ACCOUNT=
14-
SNOWFLAKE_USER=
1510
DBT_ENV_SECRET_SNOWFLAKE_PASS=
16-
SNOWFLAKE_ROLE=
17-
SNOWFLAKE_DATABASE=
18-
SNOWFLAKE_SCHEMA=
19-
SNOWFLAKE_WAREHOUSE=
11+
# local testing only
12+
# SNOWFLAKE_SCHEMA=
2013

2114
# bigquery
2215
BIGQUERY_PROJECT=
2316
BIGQUERY_SCHEMA=
2417
BIGQUERY_KEYFILE_JSON=
18+
# local testing only
19+
# BIGQUERY_SCHEMA=
2520

21+
# synapse
22+
SYNAPSE_CLIENT_SECRET=
23+
# local testing only
24+
# SYNAPSE_SCHEMA=
2625

26+
27+
# NOT CURRENTLY USED
2728
# databricks
2829
DATABRICKS_TEST_HOST=
2930
DATBRICKS_TEST_ENDPOINT=
3031
DATABRICKS_TOKEN=
3132

32-
# msft
33-
SYNAPSE_TEST_SERVER=
34-
SYNAPSE_TEST_DBNAME=
35-
SYNAPSE_TEST_USER=
36-
SYNAPSE_TEST_PASS=
3733

34+
# old
3835
AZURESQL_TEST_SERVER=
3936
AZURESQL_TEST_DBNAME=
4037
AZURESQL_TEST_USER=

‎integration_tests/vars.env.sample

+11-3
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,26 @@
11
# gh variable set -f integration_tests/vars.env
22

33
# redshift
4+
# NOTE: REDSHIFT_SPECTRUM_IAM_ROLE is currently hard-coded
45
REDSHIFT_HOST=
56
REDSHIFT_USER=
67
REDSHIFT_DATABASE=
78
REDSHIFT_PORT=
8-
REDSHIFT_SPECTRUM_IAM_ROLE=
99

1010
# snowflake
11-
SNOWFLAKE_ACCOUNT=
1211
SNOWFLAKE_USER=
1312
SNOWFLAKE_ROLE=
1413
SNOWFLAKE_DATABASE=
1514
SNOWFLAKE_WAREHOUSE=
1615

1716
# bigquery
18-
BIGQUERY_PROJECT=
17+
BIGQUERY_PROJECT=
18+
19+
# synapse
20+
SYNAPSE_DRIVER=
21+
SYNAPSE_HOST=
22+
SYNAPSE_PORT=
23+
SYNAPSE_DATABASE=
24+
SYNAPSE_AUTHENTICATION=
25+
SYNAPSE_TENANT_ID=
26+
SYNAPSE_CLIENT_ID=

‎macros/plugins/sqlserver/create_external_schema.sql ‎macros/plugins/fabric/create_external_schema.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{% macro sqlserver__create_external_schema(source_node) %}
1+
{% macro fabric__create_external_schema(source_node) %}
22
{# https://learn.microsoft.com/en-us/sql/t-sql/statements/create-schema-transact-sql?view=sql-server-ver16 #}
33

44
{% set ddl %}

‎macros/plugins/sqlserver/create_external_table.sql ‎macros/plugins/fabric/create_external_table.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{% macro sqlserver__create_external_table(source_node) %}
1+
{% macro fabric__create_external_table(source_node) %}
22

33
{%- set columns = source_node.columns.values() -%}
44
{%- set external = source_node.external -%}

‎macros/plugins/sqlserver/get_external_build_plan.sql ‎macros/plugins/fabric/get_external_build_plan.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{% macro sqlserver__get_external_build_plan(source_node) %}
1+
{% macro fabric__get_external_build_plan(source_node) %}
22

33
{% set build_plan = [] %}
44

‎macros/plugins/sqlserver/helpers/dropif.sql ‎macros/plugins/fabric/helpers/dropif.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{% macro sqlserver__dropif(node) %}
1+
{% macro fabric__dropif(node) %}
22

33
{% set ddl %}
44
if object_id ('{{source(node.source_name, node.name)}}') is not null

‎pyproject.toml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[project]
2+
name = "dbt-external-tables"
3+
version = "0.1.0"
4+
description = "Add your description here"
5+
readme = "README.md"
6+
requires-python = ">=3.11"
7+
dependencies = [
8+
"dbt-bigquery>=1.9.0",
9+
"dbt-core>=1.9.1",
10+
"dbt-redshift>=1.9.0",
11+
"dbt-snowflake>=1.9.0",
12+
"dbt-synapse>=1.8.2",
13+
"tox>=4.23.2",
14+
]

‎supported_adapters.env

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
SUPPORTED_ADAPTERS=snowflake,redshift,bigquery
1+
SUPPORTED_ADAPTERS=snowflake,redshift,bigquery,synapse

‎tox.ini

+25-1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,15 @@ passenv =
2424
BIGQUERY_KEYFILE_JSON
2525
BIGQUERY_PROJECT
2626
BIGQUERY_SCHEMA
27+
# synapse
28+
SYNAPSE_DRIVER
29+
SYNAPSE_HOST
30+
SYNAPSE_PORT
31+
SYNAPSE_DATABASE
32+
SYNAPSE_AUTHENTICATION
33+
SYNAPSE_TENANT_ID
34+
SYNAPSE_CLIENT_ID
35+
SYNAPSE_CLIENT_SECRET
2736

2837
# run dbt commands directly, assumes dbt is already installed in environment
2938
[testenv:dbt_integration_redshift]
@@ -68,4 +77,19 @@ commands =
6877
dbt run-operation prep_external --target bigquery
6978
dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --target bigquery
7079
dbt run-operation dbt_external_tables.stage_external_sources --target bigquery
71-
dbt test --target bigquery
80+
dbt test --target bigquery
81+
82+
# run dbt commands directly, assumes dbt is already installed in environment
83+
[testenv:dbt_integration_synapse]
84+
changedir = integration_tests
85+
allowlist_externals =
86+
dbt
87+
skip_install = true
88+
commands =
89+
dbt deps --target synapse
90+
dbt seed --full-refresh --target synapse
91+
dbt run --target synapse
92+
dbt run-operation prep_external --target synapse
93+
dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --target synapse
94+
dbt run-operation dbt_external_tables.stage_external_sources --target synapse
95+
dbt test --target synapse

0 commit comments

Comments
 (0)