Skip to content

Commit 80fe871

Browse files
JeanLouisLamezecJeanLouisLamezec
committed
fix(matomo): rm campaign_name frm matomo_campaign_source_medium
fix(matomo_campaign): rm matomo_content wip feat(meetings + proposals): use new macro to check column.dtype feat(proposals): add macro to deal with old_state and state rm club_utilisateurs from ci feat(ps_belge) : adding Ps Belge dbt project feat(constraints) : adding constraints and descriptive documentation to columns in demo models feat(constraints) : removing model level constraints and adding constraints packages in demo wip feat(constraints) : removing model level constraints and adding constraints packages in demo
1 parent 93b01a8 commit 80fe871

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+358
-84
lines changed

‎.github/workflows/ci_on_preprod.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ jobs:
3030
- name: Generate matrix
3131
id: set-matrix
3232
run: |
33-
project_dirs=$(find projects -maxdepth 1 -mindepth 1 -type d -not -name 'demo' -not -name 'packages' -not -name 'cea')
33+
project_dirs=$(find projects -maxdepth 1 -mindepth 1 -type d -not -name 'demo' -not -name 'packages' -not -name 'cea' -not -name 'club_utilisateurs')
3434
matrix=$(echo "$project_dirs" | jq -R -s -c 'split("\n") | map(select(length > 0)) | {project_dir: .}')
3535
echo "matrix=$matrix" >> $GITHUB_ENV
3636
echo "::set-output name=matrix::$matrix"

‎.github/workflows/ps_belge_models_run_preproduction.yml

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: ps_belge_models_run_preproduction
2+
3+
on:
4+
workflow_dispatch:
5+
6+
env:
7+
DBNAME: ${{ secrets.PS_BELGE_DBNAME_PROD }}
8+
HOST_PREPROD: ${{ secrets.HOST_PREPROD }}
9+
PASSWORD_PREPROD: ${{ secrets.PASSWORD_PREPROD }}
10+
PORT_PREPROD: ${{ secrets.PORT_PREPROD }}
11+
TARGET_SCHEMA: ${{ secrets.SCHEMA_PROD }}
12+
USER_PREPROD: ${{ secrets.USER_PREPROD }}
13+
PARTICIPATION_HOST_NAME: ${{ secrets.PS_BELGE_PARTICIPATION_HOST_NAME }}
14+
WORKING_DIRECTORY: ./projects/ps_belge
15+
16+
17+
jobs:
18+
ps_belge_models_run_preproduction:
19+
name: ps_belge_models_run_preproduction
20+
runs-on: ubuntu-latest
21+
defaults:
22+
run:
23+
working-directory: ${{ env.WORKING_DIRECTORY }}
24+
25+
steps:
26+
- name: Check out
27+
uses: actions/checkout@master
28+
29+
- name: Setup Python
30+
uses: actions/setup-python@v5
31+
with:
32+
python-version: "3.12.x"
33+
cache: 'pip'
34+
35+
- name: Install dependencies
36+
run: |
37+
pip install -r requirements.txt
38+
dbt deps --target preprod
39+
40+
- name: Run dbt models
41+
run: dbt run --target preprod
42+
43+
- name: Test dbt models
44+
run: dbt test --target preprod
45+
46+

‎.github/workflows/ps_belge_models_run_production.yml

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: ps_belge_models_run_production
2+
3+
on:
4+
workflow_dispatch:
5+
6+
env:
7+
DBNAME: ${{ secrets.PS_BELGE_DBNAME_PROD }}
8+
HOST: ${{ secrets.HOST }}
9+
PASSWORD: ${{ secrets.PASSWORD }}
10+
PORT: ${{ secrets.PORT }}
11+
TARGET_SCHEMA: ${{ secrets.SCHEMA_PROD }}
12+
USER: ${{ secrets.USER }}
13+
PARTICIPATION_HOST_NAME: ${{ secrets.PS_BELGE_PARTICIPATION_HOST_NAME }}
14+
WORKING_DIRECTORY: ./projects/ps_belge
15+
16+
17+
jobs:
18+
ps_belge_models_run:
19+
name: ps_belge_models_run_production
20+
runs-on: ubuntu-latest
21+
defaults:
22+
run:
23+
working-directory: ${{ env.WORKING_DIRECTORY }}
24+
25+
steps:
26+
- name: Check out
27+
uses: actions/checkout@master
28+
29+
- name: Setup Python
30+
uses: actions/setup-python@v5
31+
with:
32+
python-version: "3.12.x"
33+
cache: 'pip'
34+
35+
- name: Install dependencies
36+
run: |
37+
pip install -r requirements.txt
38+
dbt deps --target prod
39+
40+
- name: Run dbt models
41+
run: dbt run --target prod
42+
43+
- name: Test dbt models
44+
run: dbt test --target prod
45+
46+

‎projects/club_utilisateurs/.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
2+
target/
3+
dbt_packages/
4+
logs/

‎projects/club_utilisateurs/README.md

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
Welcome to your new dbt project!
2+
3+
### Using the starter project
4+
5+
Try running the following commands:
6+
- dbt run
7+
- dbt test
8+
9+
10+
### Resources:
11+
- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
12+
- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
13+
- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
14+
- Find [dbt events](https://events.getdbt.com) near you
15+
- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

‎projects/club_utilisateurs/analyses/.gitkeep

Whitespace-only changes.

‎projects/club_utilisateurs/config.yml

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
db_name_dev: CLUB_UTILISATEURS_DBNAME_DEV
2+
db_name_prod: CLUB_UTILISATEURS_DBNAME_PROD
3+
participation_host_name: CLUB_UTILISATEURS_PARTICIPATION_HOST_NAME

‎projects/club_utilisateurs/dbt_project.yml

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
2+
# Name your project! Project names should contain only lowercase characters
3+
# and underscores. A good package name should reflect your organization's
4+
# name or the intended use of these models
5+
name: 'club_utilisateurs'
6+
version: '1.0.0'
7+
8+
# This setting configures which "profile" dbt uses for this project.
9+
profile: 'club_utilisateurs'
10+
11+
# These configurations specify where dbt should look for different types of files.
12+
# The `model-paths` config, for example, states that models in this project can be
13+
# found in the "models/" directory. You probably won't need to change these!
14+
model-paths: ["models"]
15+
analysis-paths: ["analyses"]
16+
test-paths: ["tests"]
17+
seed-paths: ["seeds"]
18+
macro-paths: ["macros"]
19+
snapshot-paths: ["snapshots"]
20+
21+
clean-targets: # directories to be removed by `dbt clean`
22+
- "target"
23+
- "dbt_packages"
24+
25+
26+
# Configuring models
27+
# Full documentation: https://docs.getdbt.com/docs/configuring-models
28+
29+
# In this example config, we tell dbt to build all models in the example/
30+
# directory as views. These settings can be overridden in the individual model
31+
# files using the `{{ config(...) }}` macro.
32+
models:
33+
club_utilisateurs:
34+
+materialized: table

‎projects/club_utilisateurs/macros/.gitkeep

Whitespace-only changes.

‎projects/club_utilisateurs/packages.yml

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
packages:
2+
- local: ../demo
3+
- package: dbt-labs/dbt_utils
4+
version: 1.2.0

‎projects/club_utilisateurs/profiles.yml

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
club_utilisateurs:
2+
target: preprod
3+
outputs:
4+
dev:
5+
dbname: "{{ env_var('DBNAME') }}"
6+
host: "{{ env_var('HOST') }}"
7+
pass: "{{ env_var('PASSWORD') }}"
8+
port: "{{ env_var('PORT') | as_number }}"
9+
schema: "{{ env_var('TARGET_SCHEMA') }}"
10+
threads: 2
11+
type: postgres
12+
user: "{{ env_var('USER') }}"
13+
preprod:
14+
dbname: "{{ env_var('DBNAME') }}"
15+
host: "{{ env_var('HOST_PREPROD') }}"
16+
pass: "{{ env_var('PASSWORD_PREPROD') }}"
17+
port: "{{ env_var('PORT_PREPROD') | as_number }}"
18+
schema: "{{ env_var('TARGET_SCHEMA') }}"
19+
threads: 2
20+
type: postgres
21+
user: "{{ env_var('USER_PREPROD') }}"
22+
prod:
23+
dbname: "{{ env_var('DBNAME') }}"
24+
host: "{{ env_var('HOST') }}"
25+
pass: "{{ env_var('PASSWORD') }}"
26+
port: "{{ env_var('PORT') | as_number }}"
27+
schema: "{{ env_var('TARGET_SCHEMA') }}"
28+
threads: 2
29+
type: postgres
30+
user: "{{ env_var('USER') }}"

‎projects/club_utilisateurs/requirements.txt

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
dbt-postgres==1.8.2
2+
dbt-core==1.8.2

‎projects/club_utilisateurs/seeds/.gitkeep

Whitespace-only changes.

‎projects/club_utilisateurs/snapshots/.gitkeep

Whitespace-only changes.

‎projects/club_utilisateurs/tests/.gitkeep

Whitespace-only changes.

‎projects/demo/macros/check_column_type.sql

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{% macro check_column_type(relation, column_name, expected_dtype) %}
2+
{% set columns = adapter.get_columns_in_relation(relation) %}
3+
4+
{% set result = {'is_correct_type': False} %}
5+
6+
{% for column in columns %}
7+
{% if column.column == column_name %}
8+
{% if column.dtype == expected_dtype %}
9+
{% do result.update({'is_correct_type': True}) %}
10+
{% endif %}
11+
{% endif %}
12+
{% endfor %}
13+
14+
{{ return(result.is_correct_type) }}
15+
{% endmacro %}

‎projects/demo/macros/get_column_if_exists.sql

+1-3
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@
33
{% if column_name in columns | map(attribute="name") %}
44
{{ column_name }}
55
{% else %}
6-
{% if cast_type %}
7-
CAST({{ default_value }} AS {{ cast_type }}) AS {{ column_name }}
8-
{% endif %}
6+
CAST({{ default_value }} AS {{ cast_type }}) AS {{ column_name }}
97
{% endif %}
108
{% endmacro %}
119

‎projects/demo/macros/models/staging/meetings/stg_meetings_get_type_of_meetings.sql

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{% macro stg_meetings_get_type_of_meeting(relation) %}
2+
{% if check_column_type(relation, 'type_of_meeting', 'bigint') %}
3+
(CASE
4+
WHEN type_of_meeting = 0 THEN 'in_person'
5+
WHEN type_of_meeting = 10 THEN 'online'
6+
WHEN type_of_meeting = 20 THEN 'hybrid'
7+
ELSE 'unknown'
8+
END) AS type_of_meeting
9+
{% else %}
10+
type_of_meeting
11+
{% endif %}
12+
{% endmacro %}

‎projects/demo/macros/models/staging/organizations/get_name_column.sql

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{% macro get_name_column(lang='fr') %}
2+
CASE
3+
WHEN name ~ '^\s*\{.*\}\s*$' THEN (CAST(name AS jsonb)->>'{{ lang }}')
4+
ELSE name
5+
END AS name
6+
{% endmacro %}

‎projects/demo/macros/models/staging/proposals/stg_proposals_get_state.sql

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{% macro stg_proposals_get_state(relation) %}
2+
{% if check_column_type(relation, 'old_state', 'bigint') %}
3+
(CASE
4+
WHEN old_state = 0 THEN 'not_answered'
5+
WHEN old_state = 10 THEN 'evaluating'
6+
WHEN old_state = 20 THEN 'accepted'
7+
WHEN old_state = -10 THEN 'rejected'
8+
WHEN old_state = -20 THEN 'withdrawn'
9+
ELSE 'unknown'
10+
END)
11+
{% else %}
12+
state
13+
{% endif %}
14+
{% endmacro %}

‎projects/demo/models/marts/categorizations/schema.yml

-14
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,6 @@ models:
44
- name: categorizations
55
schema: "{{ env_var('TARGET_SCHEMA') }}"
66
description: "Categories and subcategories related to categorizations on the platform"
7-
8-
# Model-level constraints
9-
constraints:
10-
- type: primary_key
11-
columns: [id]
12-
- type: foreign_key
13-
columns: [decidim_category_id]
14-
expression: "{{ target.schema }}.decidim_categories (id)"
15-
- type: foreign_key
16-
columns: [categorizable_id]
17-
expression: "{{ target.schema }}.decidim_proposals (id)"
18-
- type: foreign_key
19-
columns: [categorizable_id]
20-
expression: "{{ target.schema }}.decidim_budgets (id)"
217
columns:
228
- name: id
239
description: "The unique identifier for a categorization"

‎projects/demo/models/marts/components/schema.yml

+11-12
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,16 @@ version: 2
33
models:
44
- name: components
55
description: "Model representing the components associated with participatory spaces in Decidim."
6+
67
# model-level constraints
78
constraints:
89
- type: primary_key
9-
columns: [id]
10-
- type: foreign_key
11-
columns: [ps_id]
12-
expression: "{{ target.schema }}.participatory_spaces (id)"
13-
- type: foreign_key
14-
columns: [decidim_organization_id]
15-
expression: "{{ target.schema }}.decidim_organizations (id)"
10+
columns: [ id ]
11+
- type: foreign_key # multi_column
12+
columns: [ decidim_organization_id ]
13+
to: ref('organizations')
14+
to_columns: [ id ]
15+
1616
columns:
1717
- name: id
1818
description: "Unique identifier for the component."
@@ -42,7 +42,8 @@ models:
4242
constraints:
4343
- type: not_null
4444
- type: foreign_key
45-
expression: "{{ target.schema }}.participatory_spaces (id)"
45+
to: ref('stg_decidim_participatory_processes')
46+
to_columns: [ id ]
4647
- name: ps_published_at
4748
description: "Publication date of the participatory space."
4849
- name: ps_title
@@ -59,13 +60,11 @@ models:
5960
description: "URL of the participatory space."
6061
- name: decidim_organization_id
6162
description: "Identifier of the Decidim organization."
62-
data_tests:
63-
- not_null
6463
constraints:
6564
- type: not_null
6665
- type: foreign_key
67-
expression: "{{ target.schema }}.decidim_organizations (id)"
68-
66+
to: ref('organizations')
67+
to_columns: [ id ]
6968
- name: organization_host
7069
description: "Host of the Decidim organization."
7170

‎projects/demo/models/marts/forms/schema.yml

-10
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,6 @@ version: 2
33
models:
44
- name: form_answers
55
description: "Table gathering all types of answers (short, long, optional, sorting, matrix, and file answers) submitted to forms."
6-
# Model-level constraints
7-
constraints:
8-
- type: primary_key
9-
columns: [decidim_user_id]
10-
- type: foreign_key
11-
columns: [decidim_questionnaire_id]
12-
expression: "{{ target.schema }}.forms (id)"
13-
- type: foreign_key
14-
columns: [decidim_component_id]
15-
expression: "{{ target.schema }}.components (id)"
166
columns:
177
- name: decidim_user_id
188
description: "The ID of the user who submitted the answer"

‎projects/demo/models/marts/meetings/schema.yml

-13
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,6 @@ models:
22
- name: meetings
33
schema: "{{ env_var('TARGET_SCHEMA') }}"
44
description: "Every meeting published on the platform"
5-
# model-level constraints
6-
constraints:
7-
- type: primary_key
8-
columns: [id]
9-
- type: foreign_key
10-
columns: [decidim_component_id]
11-
expression: "{{ target.schema }}.components (id)"
12-
- type: foreign_key
13-
columns: [decidim_scope_id]
14-
expression: "{{ target.schema }}.decidim_scopes (id)"
15-
- type: foreign_key
16-
columns: [decidim_author_id]
17-
expression: "{{ target.schema }}.decidim_users (id)"
185
columns:
196
- name: id
207
description: "The unique identifier for a meeting"

‎projects/demo/models/marts/proposals/schema.yml

-11
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,6 @@ version: 2
44
models:
55
- name: proposals
66
description: "Every proposal that has been created on your Decidim instance, including accepted, pending, rejected and withdrawn proposals. Does not include hidden (moderation) and unpublished proposals (i.e. linked to either an unpublished participatory process or an unpublished component)."
7-
# Model-level constraints
8-
constraints:
9-
- type: foreign_key
10-
columns: [decidim_participatory_space_id]
11-
expression: "{{ target.schema }}.participatory_spaces (id)" # Foreign key reference to participatory_spaces table
12-
- type: foreign_key
13-
columns: [decidim_component_id]
14-
expression: "{{ target.schema }}.components (id)" # Foreign key reference to components table
15-
- type: foreign_key
16-
columns: [first_author_id]
17-
expression: "{{ target.schema }}.users (id)" # Foreign key reference to users table
187
columns:
198
- name: id
209
description: "Unique identifier for the proposal"

0 commit comments

Comments
 (0)