From cccd0cffad38385d10205340a3fb248a008a1696 Mon Sep 17 00:00:00 2001 From: guykohen Date: Fri, 17 Jun 2022 15:29:49 -0400 Subject: [PATCH 1/6] Allow adding SETTINGS section through the model --- dbt/adapters/clickhouse/impl.py | 8 ++++++++ dbt/include/clickhouse/macros/adapters.sql | 1 + .../clickhouse/macros/materializations/incremental.sql | 4 ++++ dbt/include/clickhouse/macros/materializations/seed.sql | 6 ++++-- 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/clickhouse/impl.py b/dbt/adapters/clickhouse/impl.py index 4fe4c945..5eb5ae4f 100644 --- a/dbt/adapters/clickhouse/impl.py +++ b/dbt/adapters/clickhouse/impl.py @@ -253,6 +253,14 @@ def run_sql_for_tests(self, sql, fetch, conn): finally: conn.state = 'close' + @available + def get_model_settings(self, model): + settings = model['config'].get('settings', dict()) + res = [] + for key in settings: + res.append(f' {key}={settings[key]}') + return '' if len(res) == 0 else 'SETTINGS ' + ', '.join(res) + '\n' + def _expect_row_value(key: str, row: agate.Row): if key not in row.keys(): diff --git a/dbt/include/clickhouse/macros/adapters.sql b/dbt/include/clickhouse/macros/adapters.sql index 52190cf4..164f75d2 100644 --- a/dbt/include/clickhouse/macros/adapters.sql +++ b/dbt/include/clickhouse/macros/adapters.sql @@ -86,6 +86,7 @@ as ( {{ sql }} ) + {{ adapter.get_model_settings(model) }} {%- endmacro %} {% macro clickhouse__create_view_as(relation, sql) -%} diff --git a/dbt/include/clickhouse/macros/materializations/incremental.sql b/dbt/include/clickhouse/macros/materializations/incremental.sql index 8a6e5307..2a1d78f4 100644 --- a/dbt/include/clickhouse/macros/materializations/incremental.sql +++ b/dbt/include/clickhouse/macros/materializations/incremental.sql @@ -116,6 +116,7 @@ {% macro clickhouse__incremental_create(old_relation, target_relation) %} create table {{ target_relation }} as {{ old_relation }} + {%- endmacro %} {% macro clickhouse__incremental_cur_insert(old_relation, tmp_relation, target_relation, unique_key=none) %} @@ -129,6 +130,7 @@ select ({{ unique_key }}) from {{ tmp_relation }} ) + {{ adapter.get_model_settings(model) }} {%- endmacro %} {% macro clickhouse__incremental_insert_from_table(tmp_relation, target_relation) %} @@ -138,6 +140,7 @@ insert into {{ target_relation }} ({{ dest_cols_csv }}) select {{ dest_cols_csv }} from {{ tmp_relation }} + {{ adapter.get_model_settings(model) }} {%- endmacro %} {% macro clickhouse__incremental_insert(target_relation, sql) %} @@ -146,6 +149,7 @@ insert into {{ target_relation }} ({{ dest_cols_csv }}) {{ sql }} + {{ adapter.get_model_settings(model) }} {%- endmacro %} {% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %} diff --git a/dbt/include/clickhouse/macros/materializations/seed.sql b/dbt/include/clickhouse/macros/materializations/seed.sql index 2d692fe4..22c0a339 100644 --- a/dbt/include/clickhouse/macros/materializations/seed.sql +++ b/dbt/include/clickhouse/macros/materializations/seed.sql @@ -3,8 +3,10 @@ {% set data_sql = adapter.get_csv_data(agate_table) %} {% set sql -%} - insert into {{ this.render() }} ({{ cols_sql }}) format CSV - {{ data_sql }} + insert into {{ this.render() }} ({{ cols_sql }}) + {{ adapter.get_model_settings(model) }} + format CSV + {{ data_sql }} {%- endset %} {% do adapter.add_query(sql, bindings=agate_table, abridge_sql_log=True) %} From 49dd5fdf178d2ece2040a2445cc957d316b6f3ae Mon Sep 17 00:00:00 2001 From: guykohen Date: Fri, 17 Jun 2022 15:54:26 -0400 Subject: [PATCH 2/6] Run create table as command instead of as query --- .../clickhouse/macros/materializations/incremental.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbt/include/clickhouse/macros/materializations/incremental.sql b/dbt/include/clickhouse/macros/materializations/incremental.sql index 2a1d78f4..a72d676c 100644 --- a/dbt/include/clickhouse/macros/materializations/incremental.sql +++ b/dbt/include/clickhouse/macros/materializations/incremental.sql @@ -57,7 +57,9 @@ {% else %} {% set old_relation = existing_relation.incorporate(path={"identifier": old_identifier}) %} -- Create a table with only updated rows. - {% do run_query(create_table_as(False, tmp_relation, sql)) %} + {% call statement('creat_temp_table_statement') %} + {{ create_table_as(False, tmp_relation, sql) }} + {% endcall %} {% if on_schema_change != 'ignore' %} -- Update schema types if necessary. {% do adapter.expand_target_column_types( From 2b81773266412d72db8ca0130b123ebb2142c643 Mon Sep 17 00:00:00 2001 From: guykohen Date: Fri, 17 Jun 2022 16:42:15 -0400 Subject: [PATCH 3/6] Add custom settings to teh connection through the Credentials --- dbt/adapters/clickhouse/connections.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/clickhouse/connections.py b/dbt/adapters/clickhouse/connections.py index 94b97eb3..ff087f9b 100644 --- a/dbt/adapters/clickhouse/connections.py +++ b/dbt/adapters/clickhouse/connections.py @@ -2,7 +2,7 @@ import uuid from contextlib import contextmanager from dataclasses import dataclass -from typing import Any, Optional, Tuple +from typing import Any, Optional, Tuple, Dict import agate import clickhouse_connect @@ -40,6 +40,7 @@ class ClickhouseCredentials(Credentials): use_default_schema: bool = ( False # This is used in tests to make sure we connect always to the default database. ) + custom_settings: Optional[Dict[str, Any]] = None @property def type(self): @@ -103,6 +104,7 @@ def open(cls, connection): credentials = cls.get_credentials(connection.credentials) try: + custom_settings = dict() if credentials.custom_settings is None else credentials.custom_settings handle = clickhouse_connect.get_client( host=credentials.host, port=credentials.port, @@ -116,6 +118,7 @@ def open(cls, connection): verify=credentials.verify, query_limit=0, session_id='dbt::' + str(uuid.uuid4()), + **custom_settings ) connection.handle = handle connection.state = 'open' From bb0a7818522a043b481e4efe823a9ac77c16bd71 Mon Sep 17 00:00:00 2001 From: guykohen Date: Fri, 17 Jun 2022 16:48:30 -0400 Subject: [PATCH 4/6] fixing lint --- dbt/adapters/clickhouse/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/clickhouse/connections.py b/dbt/adapters/clickhouse/connections.py index ff087f9b..4a2e5416 100644 --- a/dbt/adapters/clickhouse/connections.py +++ b/dbt/adapters/clickhouse/connections.py @@ -2,7 +2,7 @@ import uuid from contextlib import contextmanager from dataclasses import dataclass -from typing import Any, Optional, Tuple, Dict +from typing import Any, Dict, Optional, Tuple import agate import clickhouse_connect From 734ad548d3c17cd1fd184406fa1c709b1ea5f7b5 Mon Sep 17 00:00:00 2001 From: guykohen Date: Fri, 17 Jun 2022 17:03:04 -0400 Subject: [PATCH 5/6] fixing lint and update version to 1.1.0.2 --- dbt/adapters/clickhouse/__version__.py | 2 +- dbt/adapters/clickhouse/connections.py | 6 ++++-- setup.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dbt/adapters/clickhouse/__version__.py b/dbt/adapters/clickhouse/__version__.py index 989f89b7..2a5cb23d 100644 --- a/dbt/adapters/clickhouse/__version__.py +++ b/dbt/adapters/clickhouse/__version__.py @@ -1 +1 @@ -version = '1.1.0.1' +version = '1.1.0.2' diff --git a/dbt/adapters/clickhouse/connections.py b/dbt/adapters/clickhouse/connections.py index 4a2e5416..82bbddb3 100644 --- a/dbt/adapters/clickhouse/connections.py +++ b/dbt/adapters/clickhouse/connections.py @@ -104,7 +104,9 @@ def open(cls, connection): credentials = cls.get_credentials(connection.credentials) try: - custom_settings = dict() if credentials.custom_settings is None else credentials.custom_settings + custom_settings = ( + dict() if credentials.custom_settings is None else credentials.custom_settings + ) handle = clickhouse_connect.get_client( host=credentials.host, port=credentials.port, @@ -118,7 +120,7 @@ def open(cls, connection): verify=credentials.verify, query_limit=0, session_id='dbt::' + str(uuid.uuid4()), - **custom_settings + **custom_settings, ) connection.handle = handle connection.state = 'open' diff --git a/setup.py b/setup.py index c8bf2d4f..60b4e8c9 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ def _dbt_clickhouse_version(): ] }, install_requires=[ - f'dbt-core=={dbt_version}', + f'dbt-core~={dbt_version}', 'clickhouse-connect>=0.1.2', ], python_requires=">=3.7", From 338ce8802fb168c9db02a38a111c5d226f2b66c8 Mon Sep 17 00:00:00 2001 From: guykohen Date: Fri, 17 Jun 2022 20:07:28 -0400 Subject: [PATCH 6/6] Updated clickhouse-connect version --- dev_requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index 9bba129f..85642181 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,5 +1,5 @@ dbt-core==1.1.0 -clickhouse-connect>=0.1.2 +clickhouse-connect>=0.1.4 pytest==7.0.0 pytest-dotenv==0.5.2 dbt-tests-adapter==1.1 diff --git a/setup.py b/setup.py index 60b4e8c9..7e545e10 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ def _dbt_clickhouse_version(): }, install_requires=[ f'dbt-core~={dbt_version}', - 'clickhouse-connect>=0.1.2', + 'clickhouse-connect>=0.1.4', ], python_requires=">=3.7", platforms='any',