diff --git a/.github/workflows/create-channel.yml b/.github/workflows/create-channel.yml index f1c70c22a..775a59eff 100644 --- a/.github/workflows/create-channel.yml +++ b/.github/workflows/create-channel.yml @@ -1,18 +1,14 @@ name: Create Channel -# need images created at least once per branch, even if there are no docker changes -# so that downstream projects can use the branch channel. on: - push: - branches-ignore: - - 'main' - paths: - - '.github/workflows/create-channel.yml' - - '.github/actions/**' - - '.github/docker-images/**' - - '.github/workflows/*.sh' - - 'builder/**' - create: + workflow_call: + secrets: + AWS_S3_BUCKET: + required: true + CRT_CI_ROLE_ARN: + required: true + AWS_ECR_REPO: + required: true env: AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} @@ -50,12 +46,6 @@ jobs: python3 -m zipapp --python="/usr/bin/env python3" -m "builder.main:main" --output=build/builder staging aws s3 cp build/builder s3://$AWS_S3_BUCKET/channels/$CHANNEL/builder.pyz - - name: Artifact builder - uses: actions/upload-artifact@v4 - with: - name: builder - path: build/builder - - name: Upload container CI script run: aws s3 cp ./.github/workflows/linux-container-ci.sh s3://aws-crt-test-stuff/ci/${{ steps.tag.outputs.release_tag }}/linux-container-ci.sh diff --git a/.github/workflows/dispatcher.sh b/.github/workflows/dispatcher.sh new file mode 100755 index 000000000..cb386703b --- /dev/null +++ b/.github/workflows/dispatcher.sh @@ -0,0 +1,67 @@ +#!/bin/bash +set -e + +# checks for previous successful runs on the branch +BRANCH="${GITHUB_REF_NAME}" +echo "Checking for previous successful runs on this branch: $BRANCH" +COMMIT_ID=$(gh run list -w="Dispatcher Workflow" --branch="$BRANCH" --json conclusion,headSha --jq 'first(.[] | select(.conclusion == "success")) | .headSha // empty' ) +if [[ -z "$COMMIT_ID" ]]; then + echo "Found no successful dispatch runs on this branch." + echo "trigger_create=true" >> $GITHUB_OUTPUT + echo "trigger_sanity_test=true" >> $GITHUB_OUTPUT + exit 0 +fi + +# check if new changes on push requires re-running the create-channel +# we look at diffs from the last successful workflow run to current commit +if ! git fetch origin $COMMIT_ID; then + echo "Failed to fetch commit $COMMIT_ID." + echo "Setting create and sanity test to true because this might be a new branch with the same name." + echo "trigger_create=true" >> $GITHUB_OUTPUT + echo "trigger_sanity_test=true" >> $GITHUB_OUTPUT + exit 0 +fi + +SHORT_HASH=$(git rev-parse --short $COMMIT_ID) +COMMIT_MESSAGE=$(git log --format="%s" -n 1 $COMMIT_ID) +echo "Found previous successful run for commit $SHORT_HASH: $COMMIT_MESSAGE" + +CHANGED="$(git diff --name-only $COMMIT_ID $GITHUB_SHA)" + +echo "---------------------" +echo "CHANGES" +echo "---------------------" +echo "$CHANGED" + +CHANGES_THAT_TRIGGER_CREATE="(^\.github/actions/.*)|"\ +"(^\.github/workflows/[^/]*\.sh$)|"\ +"(^\.github/workflows/(create-channel|dispatcher)\.yml$)|"\ +"(^\.github/docker-images/.*$)|"\ +"(^builder/.*)" + +CHANGES_THAT_TRIGGERED_CREATE=$(echo "$CHANGED" | grep -E "$CHANGES_THAT_TRIGGER_CREATE") || true # job should continue if no matches are found + +if [ -n "$CHANGES_THAT_TRIGGERED_CREATE" ]; then + echo "---------------------" + echo "CHANGES THAT TRIGGERED CREATE AND SANITY TEST" + echo "---------------------" + echo "$CHANGES_THAT_TRIGGERED_CREATE" + echo "trigger_create=true" >> $GITHUB_OUTPUT + echo "trigger_sanity_test=true" >> $GITHUB_OUTPUT +else + echo "No changes detected that would require channel-create flow to run." + echo "trigger_create=false" >> $GITHUB_OUTPUT + + CHANGES_THAT_DO_NOT_TRIGGER_SANITY_TEST="(^.*\.md$)|(^.gitignore)|(NOTICE)|(LICENSE)" + + if [[ -n $(echo "$CHANGED" | grep -vE "$CHANGES_THAT_DO_NOT_TRIGGER_SANITY_TEST") ]]; then + echo "---------------------" + echo "CHANGES THAT TRIGGER SANITY TEST" + echo "---------------------" + echo "$CHANGED" | grep -vE "$CHANGES_THAT_DO_NOT_TRIGGER_SANITY_TEST" + echo "trigger_sanity_test=true" >> $GITHUB_OUTPUT + else + echo "No changes detected that would require a sanity test." + echo "trigger_sanity_test=false" >> $GITHUB_OUTPUT + fi +fi diff --git a/.github/workflows/dispatcher.yml b/.github/workflows/dispatcher.yml new file mode 100644 index 000000000..e173e4c63 --- /dev/null +++ b/.github/workflows/dispatcher.yml @@ -0,0 +1,77 @@ +name: Dispatcher Workflow + +permissions: + contents: read + actions: read + id-token: write + +on: + push: + branches-ignore: + - 'main' + +jobs: + setup_dispatch: # This job sets up variables that determine if create and/or sanity test need to be run. + runs-on: ubuntu-latest + outputs: + trigger_create: ${{ steps.set_trigger.outputs.trigger_create }} + trigger_sanity_test: ${{ steps.set_trigger.outputs.trigger_sanity_test }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - name: Set flags for create channel and sanity test + id: set_trigger + run: './.github/workflows/dispatcher.sh' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_REF: ${{ github.ref }} + + - name: results + run: | + echo "trigger_create: ${{ steps.set_trigger.outputs.trigger_create }}" + echo "trigger_sanity_test: ${{ steps.set_trigger.outputs.trigger_sanity_test }}" + + run_create_channel: + needs: setup_dispatch + if: ${{ needs.setup_dispatch.outputs.trigger_create == 'true' }} + uses: ./.github/workflows/create-channel.yml + secrets: + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + CRT_CI_ROLE_ARN: ${{ secrets.CRT_CI_ROLE_ARN }} + AWS_ECR_REPO: ${{ secrets.AWS_ECR_REPO }} + + sanity_test_after_create_channel: + needs: [setup_dispatch, run_create_channel] + if: ${{ needs.setup_dispatch.outputs.trigger_sanity_test == 'true' && needs.setup_dispatch.outputs.trigger_create == 'true' }} + uses: ./.github/workflows/sanity-test.yml + secrets: + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + CRT_CI_ROLE_ARN: ${{ secrets.CRT_CI_ROLE_ARN }} + AWS_ECR_REPO: ${{ secrets.AWS_ECR_REPO }} + + sanity_test_skip_create_channel: + needs: setup_dispatch + if: ${{ needs.setup_dispatch.outputs.trigger_sanity_test == 'true' && needs.setup_dispatch.outputs.trigger_create != 'true' }} + uses: ./.github/workflows/sanity-test.yml + secrets: + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + CRT_CI_ROLE_ARN: ${{ secrets.CRT_CI_ROLE_ARN }} + AWS_ECR_REPO: ${{ secrets.AWS_ECR_REPO }} + + validate_tests_passed: + needs: [setup_dispatch, run_create_channel, sanity_test_after_create_channel, sanity_test_skip_create_channel] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check all jobs succeeded + run: | + for result in "${{ needs.run_create_channel.result }}" "${{ needs.sanity_test_after_create_channel.result }}" "${{ needs.sanity_test_skip_create_channel.result }}"; do + if [[ "$result" != "success" && "$result" != "skipped" ]]; then + echo "One or more jobs failed, were cancelled, or had errors (result: $result)" + exit 1 + fi + done + echo "All required jobs have completed successfully" diff --git a/.github/workflows/sanity-test.yml b/.github/workflows/sanity-test.yml index 19d749ce5..4c98e2c78 100644 --- a/.github/workflows/sanity-test.yml +++ b/.github/workflows/sanity-test.yml @@ -1,9 +1,14 @@ name: Sanity Tests on: - push: - branches-ignore: - - 'main' + workflow_call: + secrets: + AWS_S3_BUCKET: + required: true + CRT_CI_ROLE_ARN: + required: true + AWS_ECR_REPO: + required: true env: AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}