From dfd2e0e62592e2f3771aa94c438ce78d690ca1eb Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Wed, 30 Oct 2024 11:35:53 +0800 Subject: [PATCH 1/5] Testing two jobs workflow Signed-off-by: Wei-Chun, Chang --- .github/workflows/recce_ci.yml | 44 +++++++++++++++++++++++++--------- models/marts/customers.sql | 1 + 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/.github/workflows/recce_ci.yml b/.github/workflows/recce_ci.yml index c6cde9d..bb10a25 100644 --- a/.github/workflows/recce_ci.yml +++ b/.github/workflows/recce_ci.yml @@ -4,9 +4,35 @@ on: branches: [main] jobs: - check-pull-request: + prepare-artifacts: name: Prepare for Recce runs-on: ubuntu-latest + steps: + - name: Trigger the dbt cloud job and fetch artifacts for Recce + uses: datarecce/dbt-cloud-action@main + id: recce_dbt_cloud_run + with: + dbt_cloud_token: ${{ secrets.DBT_CLOUD_API_TOKEN }} + dbt_cloud_account_id: 62083 + dbt_cloud_base_job_id: 747906 + dbt_cloud_current_job_id: 747907 + failure_on_error: true + + - name: Upload DBT Artifacts for CICD job + uses: actions/upload-artifact@v4 + with: + name: target + path: target/ + + - name: Upload DBT Artifacts for Prod job + uses: actions/upload-artifact@v4 + with: + name: target-base + path: target-base/ + + recce-run: + needs: prepare-artifacts + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: @@ -20,21 +46,17 @@ jobs: - name: Install dependencies run: pip install -r requirements.txt - - name: Trigger the dbt cloud job and fetch artifacts for Recce - uses: datarecce/dbt-cloud-action@main - id: recce_dbt_cloud_run - with: - dbt_cloud_token: ${{ secrets.DBT_CLOUD_API_TOKEN }} - dbt_cloud_account_id: 62083 - dbt_cloud_base_job_id: 747906 - dbt_cloud_current_job_id: 747907 - failure_on_error: true - - uses: "google-github-actions/auth@v2" id: google-auth with: credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY_JSON }} + - name: Download artifactss + uses: actions/download-artifact@v4 + + - name: Update dbt dependencies + run: dbt deps + - name: Run Recce in cloud mode env: BQ_PROJECT: ${{ steps.google-auth.outputs.project_id }} diff --git a/models/marts/customers.sql b/models/marts/customers.sql index 129eb8f..c0f603f 100644 --- a/models/marts/customers.sql +++ b/models/marts/customers.sql @@ -42,6 +42,7 @@ joined as ( customer_orders_summary.lifetime_spend_pretax, customer_orders_summary.lifetime_tax_paid, customer_orders_summary.lifetime_spend, + customer_orders_summary.lifetime_spend as dummy, case when customer_orders_summary.is_repeat_buyer then 'returning' From 68e84dd4d0b1269571320881fd833728c6a0c6f3 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Wed, 30 Oct 2024 12:01:02 +0800 Subject: [PATCH 2/5] Add an dummy customer record Signed-off-by: Wei-Chun, Chang --- jaffle-data/raw_customers.csv | 1 + models/marts/customers.sql | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/jaffle-data/raw_customers.csv b/jaffle-data/raw_customers.csv index 7c020d0..3ac87ef 100644 --- a/jaffle-data/raw_customers.csv +++ b/jaffle-data/raw_customers.csv @@ -934,3 +934,4 @@ e4d6980c-4b68-4255-97a5-2540f5ef35bc,Kimberly Schneider 9f72b282-1f70-450b-988f-9b3029fa121e,Christina Hall 4c11a926-7100-4d06-8910-5b0d5f3d7e9a,Jaime Hester 099dd9b8-0a76-482f-92e1-f754d25d5aee,Krista Young +48aed6a8-2d01-43cd-a3c9-070fe52ff8c0,Random Dummy diff --git a/models/marts/customers.sql b/models/marts/customers.sql index c0f603f..129eb8f 100644 --- a/models/marts/customers.sql +++ b/models/marts/customers.sql @@ -42,7 +42,6 @@ joined as ( customer_orders_summary.lifetime_spend_pretax, customer_orders_summary.lifetime_tax_paid, customer_orders_summary.lifetime_spend, - customer_orders_summary.lifetime_spend as dummy, case when customer_orders_summary.is_repeat_buyer then 'returning' From f550650cf7989a602d99f25f6573ee02c2f6558e Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Wed, 30 Oct 2024 12:07:07 +0800 Subject: [PATCH 3/5] Change criteria Signed-off-by: Wei-Chun, Chang --- jaffle-data/raw_customers.csv | 1 - models/marts/customers.sql | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/jaffle-data/raw_customers.csv b/jaffle-data/raw_customers.csv index 3ac87ef..7c020d0 100644 --- a/jaffle-data/raw_customers.csv +++ b/jaffle-data/raw_customers.csv @@ -934,4 +934,3 @@ e4d6980c-4b68-4255-97a5-2540f5ef35bc,Kimberly Schneider 9f72b282-1f70-450b-988f-9b3029fa121e,Christina Hall 4c11a926-7100-4d06-8910-5b0d5f3d7e9a,Jaime Hester 099dd9b8-0a76-482f-92e1-f754d25d5aee,Krista Young -48aed6a8-2d01-43cd-a3c9-070fe52ff8c0,Random Dummy diff --git a/models/marts/customers.sql b/models/marts/customers.sql index 129eb8f..30736fd 100644 --- a/models/marts/customers.sql +++ b/models/marts/customers.sql @@ -18,7 +18,7 @@ customer_orders_summary as ( orders.customer_id, count(distinct orders.order_id) as count_lifetime_orders, - count(distinct orders.order_id) > 1 as is_repeat_buyer, + count(distinct orders.order_id) > 2 as is_repeat_buyer, min(orders.ordered_at) as first_ordered_at, max(orders.ordered_at) as last_ordered_at, sum(orders.subtotal) as lifetime_spend_pretax, From 155ea5af8e8c5d0dbd54e8e0067d1bf4ee89d6ea Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Wed, 30 Oct 2024 12:22:58 +0800 Subject: [PATCH 4/5] Add commit sha Signed-off-by: Wei-Chun, Chang --- .github/workflows/recce_ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/recce_ci.yml b/.github/workflows/recce_ci.yml index bb10a25..d82a118 100644 --- a/.github/workflows/recce_ci.yml +++ b/.github/workflows/recce_ci.yml @@ -17,6 +17,7 @@ jobs: dbt_cloud_base_job_id: 747906 dbt_cloud_current_job_id: 747907 failure_on_error: true + git_sha: ${{ github.event.pull_request.head.sha }} - name: Upload DBT Artifacts for CICD job uses: actions/upload-artifact@v4 From 802a3b3c202464164a98c26e8d7c1115726b559e Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Tue, 26 Nov 2024 00:23:33 +0800 Subject: [PATCH 5/5] Remove commit sha to dbt cloud action Signed-off-by: Wei-Chun, Chang --- .github/workflows/recce_ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/recce_ci.yml b/.github/workflows/recce_ci.yml index d82a118..bb10a25 100644 --- a/.github/workflows/recce_ci.yml +++ b/.github/workflows/recce_ci.yml @@ -17,7 +17,6 @@ jobs: dbt_cloud_base_job_id: 747906 dbt_cloud_current_job_id: 747907 failure_on_error: true - git_sha: ${{ github.event.pull_request.head.sha }} - name: Upload DBT Artifacts for CICD job uses: actions/upload-artifact@v4