From e79f21b42c8dd80d16b98693368366299855691a Mon Sep 17 00:00:00 2001 From: David Baussart Date: Fri, 15 Sep 2023 18:18:58 +0200 Subject: [PATCH 1/5] Load template from dbt_invoke_template.yml --- dbt_invoke/internal/_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbt_invoke/internal/_utils.py b/dbt_invoke/internal/_utils.py index 7f69a7e..d66d1b6 100644 --- a/dbt_invoke/internal/_utils.py +++ b/dbt_invoke/internal/_utils.py @@ -132,6 +132,10 @@ def get_project_info(ctx, project_dir=None): Path(project_path, macro_path) for macro_path in project_yml.get('macro-paths', ['macros']) ] + # retrieves template information if provided + template_path = Path(project_path, 'dbt_invoke_template.yml') + if template_path.exists(): + ctx.config['template_yml'] = parse_yaml(template_path) # Set context config key-value pairs ctx.config['project_path'] = project_path ctx.config['project_name'] = project_name From 9e8d9c56683070d8f5c34beea81d06e0f2734220 Mon Sep 17 00:00:00 2001 From: David Baussart Date: Fri, 15 Sep 2023 20:21:35 +0200 Subject: [PATCH 2/5] Apply template on property file update --- dbt_invoke/properties.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/dbt_invoke/properties.py b/dbt_invoke/properties.py index 01412e1..20caee4 100644 --- a/dbt_invoke/properties.py +++ b/dbt_invoke/properties.py @@ -715,6 +715,7 @@ def _create_property_file( property_path, resource_dict, columns, + ctx.config['template_yml'] ) _utils.write_yaml( property_path, @@ -786,7 +787,12 @@ def _get_columns(ctx, resource_location, resource_dict, **kwargs): return columns -def _structure_property_file_dict(location, resource_dict, columns_list): +def _structure_property_file_dict( + location, + resource_dict, + columns_list, + template_yml +): """ Structure a dictionary that will be used to create a property file @@ -813,6 +819,12 @@ def _structure_property_file_dict(location, resource_dict, columns_list): property_file_dict = _get_property_header(resource_name, resource_type) # Get the sub-dictionaries of each existing column resource_type_plural = _SUPPORTED_RESOURCE_TYPES[resource_type] + # If dbt_invoke_template.yml exists, adds templated properties to header + # when not already present + if template_yml and resource_type in template_yml: + _apply_template( + property_file_dict[resource_type_plural][0], template_yml[resource_type] + ) existing_columns_dict = { item['name']: item for item in property_file_dict[resource_type_plural][0]['columns'] @@ -825,12 +837,33 @@ def _structure_property_file_dict(location, resource_dict, columns_list): column_dict = existing_columns_dict.get( column, _get_property_column(column) ) + # If dbt_invoke_template.yml exists, adds templated properties to column + # when not already present + if template_yml and 'columns' in template_yml.get(resource_type): + _apply_template( + column_dict, template_yml[resource_type]['columns'] + ) property_file_dict[resource_type_plural][0]['columns'].append( column_dict ) return property_file_dict +def _apply_template(property_dict, template): + """ + Updates a dictionary with template's yml default properties + if said dictionary does not already contain the properties. + + :param property_dict: the dictionary to be updated + :param template: a dictionary representing the templated properties + to be applied on property_dict + :return: None + """ + for key in template: + if key not in property_dict: + property_dict[key] = template[key] + + def _get_property_header(resource, resource_type, properties=None): """ Create a dictionary representing resources properties From 25ef8c5e672a19ccc955202d38b53c33c7815e52 Mon Sep 17 00:00:00 2001 From: David Baussart Date: Fri, 15 Sep 2023 20:21:43 +0200 Subject: [PATCH 3/5] dbt_invoke_template.yml documentation --- README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/README.md b/README.md index 071ee53..b25c57c 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,42 @@ dbt-invoke properties.delete - `` uses the same arguments as for creating/updating property files, except for `--threads`. + +### Applying default properties using `dbt_invoke_template.yml` + +You can pass custom default properties to be added to property files on update. +To do so, add a `dbt_invoke_template.yml` at the root of your dbt project, +using the following convention: + + +``` +model: + meta: + owner: "@default" + columns: + foo: "bar" + +seed: + meta: + owner: "@default" + columns: + foo: "bar" + +snapshot: + meta: + owner: "@default" + columns: + foo: "bar" + +analysis: + meta: + owner: "@default" + columns: + foo: "bar" +``` + +Note: only `model`, `seed`, `snapshot` and `analysis` are supported. + ### Help - To view the list of available commands and their short descriptions, run: From 5c502e362aa4b2b189d6379a7f3ec7364e80367c Mon Sep 17 00:00:00 2001 From: david-whimsical <115995643+david-whimsical@users.noreply.github.com> Date: Wed, 11 Oct 2023 10:53:29 +0200 Subject: [PATCH 4/5] making sure template_yml exists in config if the key isn't present, it will break what's in dbt_invoke/properties.py Co-authored-by: Robert Astel --- dbt_invoke/internal/_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbt_invoke/internal/_utils.py b/dbt_invoke/internal/_utils.py index d66d1b6..c86cddb 100644 --- a/dbt_invoke/internal/_utils.py +++ b/dbt_invoke/internal/_utils.py @@ -136,6 +136,8 @@ def get_project_info(ctx, project_dir=None): template_path = Path(project_path, 'dbt_invoke_template.yml') if template_path.exists(): ctx.config['template_yml'] = parse_yaml(template_path) + else: + ctx.config['template_yml'] = None # Set context config key-value pairs ctx.config['project_path'] = project_path ctx.config['project_name'] = project_name From d3db8693d705fb1a3b378053cae03b03fd6bdb39 Mon Sep 17 00:00:00 2001 From: david-whimsical <115995643+david-whimsical@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:58:31 +0100 Subject: [PATCH 5/5] resource_type not in template Co-authored-by: Robert Astel --- dbt_invoke/properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_invoke/properties.py b/dbt_invoke/properties.py index 20caee4..5e77e32 100644 --- a/dbt_invoke/properties.py +++ b/dbt_invoke/properties.py @@ -839,7 +839,7 @@ def _structure_property_file_dict( ) # If dbt_invoke_template.yml exists, adds templated properties to column # when not already present - if template_yml and 'columns' in template_yml.get(resource_type): + if template_yml and 'columns' in template_yml.get(resource_type, dict()): _apply_template( column_dict, template_yml[resource_type]['columns'] )