feat(constraints) : adding constraints and descriptive documentation to columns in demo models

fix(matomo): rm campaign_name frm matomo_campaign_source_medium

fix(matomo_campaign): rm matomo_content

wip

feat(meetings + proposals): use new macro to check column.dtype

feat(proposals): add macro to deal with old_state and state

rm club_utilisateurs from ci

feat(ps_belge) : adding Ps Belge dbt project

feat(constraints) : adding constraints and descriptive documentation to columns in demo models

feat(constraints) : removing model level constraints and adding constraints packages in demo

wip

feat(constraints) : removing model level constraints and adding constraints packages in demo

Author: Fujistone

projects/demo/models/marts/categorizations/schema.yml

@@ -1,3 +1,33 @@
+version: 2
+
+models:
+  - name: categorizations
+    schema: "{{ env_var('TARGET_SCHEMA') }}"
+    description: "Categories and subcategories related to categorizations on the platform"
+    columns:
+      - name: id
+        description: "The unique identifier for a categorization"
+        constraints:
+          - type: not_null
+          - type: unique
+          - type: primary_key
+      - name: category_name
+        description: "The name of the category"
+      - name: child_id
+        description: "The ID of the child category (if applicable)"
+      - name: child_name
+        description: "The name of the child category (if applicable)"
+      - name: categorizable_id
+        description: "The ID of the resource being categorized (could be proposal or budget)"
+        constraints:
+          - type: not_null
+          - type: foreign_key
+            expression: "{{ target.schema }}.decidim_proposals (id)"
+          - type: not_null
+          - type: foreign_key
+            expression: "{{ target.schema }}.decidim_budgets (id)"
+      - name: categorizable_type
+        description: "The type of resource being categorized (could be proposal or budget)"
 unit_tests:
   - name: testing_that_main_categories_are_properly_processed
     description: "Checks that main categories has the correct infos "

projects/demo/models/marts/components/schema.yml

@@ -3,12 +3,26 @@ version: 2
 models:
   - name: components
     description: "Model representing the components associated with participatory spaces in Decidim."
+
+    # model-level constraints
+    constraints:
+      - type: primary_key
+        columns: [ id ]
+      - type: foreign_key # multi_column
+        columns: [ decidim_organization_id ]
+        to: ref('organizations')
+        to_columns: [ id ]
+
     columns:
       - name: id
         description: "Unique identifier for the component."
         data_tests:
           - not_null
           - unique
+        constraints:
+          - type: not_null
+          - type: unique
+          - type: primary_key
       - name: manifest_name
         description: "Name of the component's manifest file. The manifest defines the component's configuration and metadata."
         data_tests:
@@ -25,6 +39,11 @@ models:
         description: "URL for accessing the component."
       - name: ps_id
         description: "Identifier of the associated participatory space."
+        constraints:
+          - type: not_null
+          - type: foreign_key
+            to: ref('stg_decidim_participatory_processes')
+            to_columns: [ id ]
       - name: ps_published_at
         description: "Publication date of the participatory space."
       - name: ps_title
@@ -41,8 +60,11 @@ models:
         description: "URL of the participatory space."
       - name: decidim_organization_id
         description: "Identifier of the Decidim organization."
-        data_tests:
-          - not_null
+        constraints:
+          - type: not_null
+          - type: foreign_key
+            to: ref('organizations')
+            to_columns: [ id ]
       - name: organization_host
         description: "Host of the Decidim organization."
 

projects/demo/models/marts/forms/schema.yml

@@ -1,8 +1,44 @@
 version: 2
 
 models:
-  - name: forms_answers
-    description: "Table gathering all types of answers (long, short, optional, sorting and matrix). "
+  - name: form_answers
+    description: "Table gathering all types of answers (short, long, optional, sorting, matrix, and file answers) submitted to forms."
     columns:
+      - name: decidim_user_id
+        description: "The ID of the user who submitted the answer"
+        constraints:
+          - type: not_null
+          - type: unique
+          - type: primary_key
+      - name: session_token
+        description: "Session token for tracking the user's session"
+      - name: position
+        description: "The position of the answer in the form"
+      - name: decidim_questionnaire_id
+        description: "The ID of the form/questionnaire to which this answer belongs"
+        constraints:
+          - type: not_null
+          - type: foreign_key
+            expression: "{{ target.schema }}.forms (id)"
+      - name: ip_hash
+        description: "IP hash for tracking the user's IP address"
+      - name: question_type
+        description: "The type of the question (e.g., short, long, matrix)"
+      - name: question_title
+        description: "The title of the question being answered"
+      - name: answer
+        description: "The answer given by the user"
+      - name: sub_matrix_question
+        description: "The question for a sub-matrix (if applicable)"
+      - name: custom_body
+        description: "Custom text associated with the answer (if applicable)"
+      - name: sorting_position
+        description: "The sorting position of the answer (sorting-type questions only)"
+      - name: form_title
+        description: "The title of the form/questionnaire"
       - name: sorting_points
-        description: "In order to evaluate the weight of the sorting answers, points are given to the answer according to its sorting position."
+        description: "Calculated sorting points based on the sorting position"
+      - name: created_at
+        description: "Timestamp of when the answer was created"
+      - name: author_status
+        description: "Status of the author "

projects/demo/models/marts/meetings/schema.yml

@@ -5,6 +5,71 @@ models:
     columns:
       - name: id
         description: "The unique identifier for a meeting"
+        data_tests:
+          - not_null
+          - unique
+        constraints:
+          - type: not_null
+          - type: unique
+          - type: primary_key
+      - name: title
+        description: "The title of the meeting"
+        data_tests:
+          - not_null
+        constraints:
+          - type: not_null
+      - name: description
+        description: "A brief description of the meeting"
+      - name: address
+        description: "The physical or virtual address of the meeting"
+      - name: attendees_count
+        description: "The total number of attendees at the meeting"
+      - name: created_at
+        description: "The timestamp when the meeting was created"
+        data_tests:
+          - not_null
+        constraints:
+          - type: not_null
+      - name: decidim_scope_id
+        description: "The identifier of the scope this meeting belongs to"
+        constraints:
+          - type: not_null
+          - type: foreign_key
+            expression: "{{ target.schema }}.decidim_scopes (id)"
+      - name: decidim_component_id
+        description: "The identifier of the component associated with the meeting"
+        constraints:
+          - type: not_null
+          - type: foreign_key
+            expression: "{{ target.schema }}.components (id)"
+      - name: start_time
+        description: "The start time of the meeting"
+      - name: end_time
+        description: "The end time of the meeting"
+      - name: registration_url
+        description: "URL for registration for the meeting"
+      - name: type_of_meeting
+        description: "The type/category of the meeting"
+      - name: translated_type_of_meeting
+        description: "The translated type of the meeting"
+      - name: private_meeting
+        description: "Indicates if the meeting is private"
+      - name: decidim_author_id
+        description: "The ID of the author or organizer of the meeting"
+        constraints:
+          - type: not_null
+          - type: foreign_key
+            expression: "{{ target.schema }}.decidim_users (id)"
+      - name: resource_type
+        description: "The type of resource related to the meeting"
+      - name: meeting_url
+        description: "The URL to access the meeting"
+      - name: categories
+        description: "Categories associated with the meeting"
+      - name: sub_categories
+        description: "Subcategories associated with the meeting"
+      - name: categorizations
+        description: "The full categorization data (raw) for the meeting"
 unit_tests:
   - name: test_url_concatenation
     description: "Check URL concatenation logic"

projects/demo/models/marts/proposals/schema.yml

@@ -10,8 +10,15 @@ models:
         tests:
           - unique
           - not_null
+        constraints:
+          - type: unique
+          - type: not_null
       - name: decidim_participatory_space_id
         description: "Identifier for the participatory space where the proposal was created"
+        constraints:
+          - type: not_null
+          - type: foreign_key
+            expression: "{{ target.schema }}.participatory_spaces (id)"  # Foreign key reference to participatory_spaces table
       - name: decidim_participatory_space_slug
         description: "Slug for the participatory space where the proposal was created"
       - name: decidim_scope_name
@@ -26,6 +33,10 @@ models:
         description: "URL link to the proposal on the Decidim platform"
       - name: decidim_component_id
         description: "Identifier for the component within which the proposal was created"
+        constraints:
+          - type: not_null
+          - type: foreign_key
+            expression: "{{ target.schema }}.components (id)"  # Foreign key reference to components table
       - name: created_at
         description: "Timestamp when the proposal was created"
       - name: published_at
@@ -38,6 +49,10 @@ models:
         description: "Array of IDs of the authors of the proposal"
       - name: first_author_id
         description: "ID of the first author of the proposal"
+        constraints:
+          - type: not_null
+          - type: foreign_key
+            expression: "{{ target.schema }}.users (id)"  # Foreign key reference to users table
       - name: address
         description: "Address related to the proposal, if any"
       - name: categories

projects/demo/packages.yml

@@ -2,4 +2,6 @@ packages:
  - package: dbt-labs/dbt_utils
    version: 1.2.0
  - package: calogica/dbt_date
-   version: 0.10.1
+   version: 0.10.1
+ - package: Snowflake-Labs/dbt_constraints
+   version: 1.0.4