Fix Synchronization of Create Channel & Sanity Tests

.github/workflows/create-channel.yml

@@ -1,18 +1,14 @@
 name: Create Channel
 
-# need images created at least once per branch, even if there are no docker changes
-# so that downstream projects can use the branch channel.
 on:
-  push:
-    branches-ignore:
-      - 'main'
-    paths:
-      - '.github/workflows/create-channel.yml'
-      - '.github/actions/**'
-      - '.github/docker-images/**'
-      - '.github/workflows/*.sh'
-      - 'builder/**'
-  create:
+  workflow_call:
+    secrets:
+      AWS_S3_BUCKET: 
+        required: true
+      CRT_CI_ROLE_ARN:
+        required: true
+      AWS_ECR_REPO: 
+        required: true
 
 env:
   AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
@@ -50,12 +46,6 @@ jobs:
         python3 -m zipapp --python="/usr/bin/env python3" -m "builder.main:main" --output=build/builder staging
         aws s3 cp build/builder s3://$AWS_S3_BUCKET/channels/$CHANNEL/builder.pyz
 
-    - name: Artifact builder
-      uses: actions/upload-artifact@v4
-      with:
-        name: builder
-        path: build/builder
-
     - name: Upload container CI script
       run: aws s3 cp ./.github/workflows/linux-container-ci.sh s3://aws-crt-test-stuff/ci/${{ steps.tag.outputs.release_tag }}/linux-container-ci.sh
 

.github/workflows/dispatcher.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+set -e
+
+# checks for previous successful runs on the branch
+BRANCH="${GITHUB_REF_NAME}"
+echo "Checking for previous successful runs on this branch: $BRANCH"
+COMMIT_ID=$(gh run list -w="Dispatcher Workflow" --branch="$BRANCH" --json conclusion,headSha --jq 'first(.[] | select(.conclusion == "success")) | .headSha // empty' )
+if [[ -z "$COMMIT_ID" ]]; then
+    echo "Found no successful dispatch runs on this branch."
+    echo "trigger_create=true" >> $GITHUB_OUTPUT
+    echo "trigger_sanity_test=true" >> $GITHUB_OUTPUT
+    exit 0
+fi
+
+# check if new changes on push requires re-running the create-channel
+# we look at diffs from the last successful workflow run to current commit
+if ! git fetch origin $COMMIT_ID; then
+    echo "Failed to fetch commit $COMMIT_ID."
+    echo "Setting create and sanity test to true because this might be a new branch with the same name."
+    echo "trigger_create=true" >> $GITHUB_OUTPUT
+    echo "trigger_sanity_test=true" >> $GITHUB_OUTPUT
+    exit 0
+fi
+
+SHORT_HASH=$(git rev-parse --short $COMMIT_ID)
+COMMIT_MESSAGE=$(git log --format="%s" -n 1 $COMMIT_ID)
+echo "Found previous successful run for commit $SHORT_HASH: $COMMIT_MESSAGE"
+
+CHANGED="$(git diff --name-only $COMMIT_ID $GITHUB_SHA)"
+
+echo "---------------------"
+echo "CHANGES"
+echo "---------------------"
+echo "$CHANGED"
+
+CHANGES_THAT_TRIGGER_CREATE="(^\.github/actions/.*)|"\
+"(^\.github/workflows/[^/]*\.sh$)|"\
+"(^\.github/workflows/(create-channel|dispatcher)\.yml$)|"\
+"(^\.github/docker-images/.*$)|"\
+"(^builder/.*)"
+
+CHANGES_THAT_TRIGGERED_CREATE=$(echo "$CHANGED" | grep -E "$CHANGES_THAT_TRIGGER_CREATE") || true # job should continue if no matches are found
+
+if [ -n "$CHANGES_THAT_TRIGGERED_CREATE" ]; then
+    echo "---------------------"
+    echo "CHANGES THAT TRIGGERED CREATE AND SANITY TEST"
+    echo "---------------------"
+    echo "$CHANGES_THAT_TRIGGERED_CREATE"
+    echo "trigger_create=true" >> $GITHUB_OUTPUT
+    echo "trigger_sanity_test=true" >> $GITHUB_OUTPUT
+else
+    echo "No changes detected that would require channel-create flow to run."
+    echo "trigger_create=false" >> $GITHUB_OUTPUT
+
+    CHANGES_THAT_DO_NOT_TRIGGER_SANITY_TEST="(^.*\.md$)|(^.gitignore)|(NOTICE)|(LICENSE)"
+
+    if [[ -n $(echo "$CHANGED" | grep -vE "$CHANGES_THAT_DO_NOT_TRIGGER_SANITY_TEST") ]]; then
+        echo "---------------------"
+        echo "CHANGES THAT TRIGGER SANITY TEST"
+        echo "---------------------"
+        echo "$CHANGED" | grep -vE "$CHANGES_THAT_DO_NOT_TRIGGER_SANITY_TEST"
+        echo "trigger_sanity_test=true" >> $GITHUB_OUTPUT
+    else
+        echo "No changes detected that would require a sanity test."
+        echo "trigger_sanity_test=false" >> $GITHUB_OUTPUT
+    fi
+fi

.github/workflows/dispatcher.yml

@@ -0,0 +1,77 @@
+name: Dispatcher Workflow
+
+permissions:
+  contents: read
+  actions: read
+  id-token: write
+
+on:
+  push:
+    branches-ignore:
+      - 'main'
+
+jobs:
+  setup_dispatch: # This job sets up variables that determine if create and/or sanity test need to be run.
+    runs-on: ubuntu-latest
+    outputs:
+      trigger_create: ${{ steps.set_trigger.outputs.trigger_create }}
+      trigger_sanity_test: ${{ steps.set_trigger.outputs.trigger_sanity_test }}
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set flags for create channel and sanity test
+        id: set_trigger
+        run: './.github/workflows/dispatcher.sh'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_SHA: ${{ github.sha }}
+          GITHUB_REF: ${{ github.ref }}
+
+      - name: results
+        run: |
+          echo "trigger_create: ${{ steps.set_trigger.outputs.trigger_create }}"
+          echo "trigger_sanity_test: ${{ steps.set_trigger.outputs.trigger_sanity_test }}"
+
+  run_create_channel:
+    needs: setup_dispatch
+    if: ${{ needs.setup_dispatch.outputs.trigger_create == 'true' }}
+    uses: ./.github/workflows/create-channel.yml
+    secrets:
+      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+      CRT_CI_ROLE_ARN: ${{ secrets.CRT_CI_ROLE_ARN }}
+      AWS_ECR_REPO: ${{ secrets.AWS_ECR_REPO }}
+
+  sanity_test_after_create_channel:
+    needs: [setup_dispatch, run_create_channel]
+    if: ${{ needs.setup_dispatch.outputs.trigger_sanity_test == 'true' && needs.setup_dispatch.outputs.trigger_create == 'true' }}
+    uses: ./.github/workflows/sanity-test.yml
+    secrets:
+      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+      CRT_CI_ROLE_ARN: ${{ secrets.CRT_CI_ROLE_ARN }}
+      AWS_ECR_REPO: ${{ secrets.AWS_ECR_REPO }}
+
+  sanity_test_skip_create_channel:
+    needs: setup_dispatch
+    if: ${{ needs.setup_dispatch.outputs.trigger_sanity_test == 'true' && needs.setup_dispatch.outputs.trigger_create != 'true' }}
+    uses: ./.github/workflows/sanity-test.yml
+    secrets:
+      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+      CRT_CI_ROLE_ARN: ${{ secrets.CRT_CI_ROLE_ARN }}
+      AWS_ECR_REPO: ${{ secrets.AWS_ECR_REPO }}
+
+  validate_tests_passed:
+    needs: [setup_dispatch, run_create_channel, sanity_test_after_create_channel, sanity_test_skip_create_channel]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check all jobs succeeded
+        run: |
+          for result in "${{ needs.run_create_channel.result }}" "${{ needs.sanity_test_after_create_channel.result }}" "${{ needs.sanity_test_skip_create_channel.result }}"; do
+            if [[ "$result" != "success" && "$result" != "skipped" ]]; then
+              echo "One or more jobs failed, were cancelled, or had errors (result: $result)"
+              exit 1
+            fi
+          done
+          echo "All required jobs have completed successfully"

.github/workflows/sanity-test.yml

@@ -1,9 +1,14 @@
 name: Sanity Tests
 
 on:
-  push:
-    branches-ignore:
-      - 'main'
+  workflow_call:
+    secrets:
+      AWS_S3_BUCKET:
+        required: true
+      CRT_CI_ROLE_ARN:
+        required: true
+      AWS_ECR_REPO:
+        required: true
 
 env:
   AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}