Update (base update)

[ghstack-poisoned]

Author: Konrad Kleine

.github/workflows/check-snapshots.yml

@@ -14,8 +14,8 @@ on:
         required: true
         type: string
       lookback_days:
-        description: "A JSON list of integers for how many days to look back. \"[0]\" means just today."
-        default: "[0]"
+        description: "A space separated list of integers for how many days to look back. \"0\" means just today."
+        default: "0"
         type: string
       strategy:
           description: 'build strategy'
@@ -39,11 +39,25 @@ jobs:
       mymatrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
     - uses: actions/checkout@v4
-    - uses: ./.github/actions/generate-matrix
+    - uses: ./.github/actions/prepare-python
+    - name: Setup Copr config file
+      env:
+        # You need to have those secrets in your repo.
+        # See also: https://copr.fedorainfracloud.org/api/.
+        COPR_CONFIG: ${{ secrets.COPR_CONFIG }}
+      run: |
+        mkdir -p ~/.config
+        echo "$COPR_CONFIG" > ~/.config/copr
+    - name: generate-matrix
       id: set-matrix
-      with:
-        strategy: ${{ inputs.strategy }}
-        lookback_days: ${{ inputs.lookback_days }}
+      run: |
+        # shellcheck disable=SC2086
+        echo "matrix=$(python3 \
+          snapshot_manager/main.py github-matrix \
+          --strategy ${{ inputs.strategy }} \
+          --lookback-days ${{ inputs.lookback_days }} \
+        )" >> $GITHUB_OUTPUT
+
   check-snapshot:
     if: github.repository_owner == 'fedora-llvm-team'
     needs: generate-matrix
@@ -57,10 +71,10 @@ jobs:
         env:
           # You need to have those secrets in your repo.
           # See also: https://copr.fedorainfracloud.org/api/.
-          COPR_CONFIG_FILE: ${{ secrets.COPR_CONFIG }}
+          COPR_CONFIG: ${{ secrets.COPR_CONFIG }}
         run: |
           mkdir -p ~/.config
-          echo "$COPR_CONFIG_FILE" > ~/.config/copr
+          echo "$COPR_CONFIG" > ~/.config/copr
 
       - name: Install Copr CLI
         if: github.event_name != 'workflow_dispatch' || (matrix.today_minus_n_days == 0 && inputs.strategy == matrix.name)
@@ -75,18 +89,11 @@ jobs:
 
       - name: "Check day (today minus ${{matrix.today_minus_n_days}} days)"
         if: github.event_name != 'workflow_dispatch' || (matrix.today_minus_n_days == 0 && inputs.strategy == matrix.name)
-        shell: bash -e {0}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TESTING_FARM_API_TOKEN_PUBLIC_RANCH: ${{ secrets.TESTING_FARM_API_TOKEN_PUBLIC_RANCH }}
           TESTING_FARM_API_TOKEN_REDHAT_RANCH: ${{ secrets.TESTING_FARM_API_TOKEN_REDHAT_RANCH }}
         run: |
-          extra_args=""
-
-          if [[ -n "${{ matrix.chroot_pattern }}" ]]; then
-            extra_args="${extra_args} --chroot-pattern ${{matrix.chroot_pattern}}"
-          fi
-
           if [[ "${{github.event_name}}" == "workflow_dispatch" ]]; then
             yyyymmdd=${{inputs.yyyymmdd}}
           else
@@ -96,12 +103,6 @@ jobs:
           # shellcheck disable=SC2086
           python3 snapshot_manager/main.py \
             --github-repo "${GITHUB_REPOSITORY}" \
-            --github-token-env GITHUB_TOKEN \
-            check ${extra_args} \
-            --maintainer-handle ${{matrix.maintainer_handle}} \
-            --packages llvm \
-            --build-strategy ${{matrix.name}} \
-            --copr-ownername ${{matrix.copr_ownername}} \
-            --copr-project-tpl ${{matrix.copr_project_tpl}} \
-            --copr-monitor-tpl ${{matrix.copr_monitor_tpl}} \
+            check \
+            --strategy ${{matrix.name}} \
             --yyyymmdd "$yyyymmdd"

.github/workflows/fedora-copr-build.yml

@@ -26,11 +26,23 @@ jobs:
       mymatrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
     - uses: actions/checkout@v4
-    - uses: ./.github/actions/generate-matrix
+    - uses: ./.github/actions/prepare-python
+    - name: Setup Copr config file
+      env:
+        # You need to have those secrets in your repo.
+        # See also: https://copr.fedorainfracloud.org/api/.
+        COPR_CONFIG: ${{ secrets.COPR_CONFIG }}
+      run: |
+        mkdir -p ~/.config
+        echo "$COPR_CONFIG" > ~/.config/copr
+    - name: generate-matrix
       id: set-matrix
-      with:
-        strategy: ${{ inputs.strategy }}
-        lookback_days: "[0]"
+      run: |
+        # shellcheck disable=SC2086
+        echo "matrix=$(python3 \
+          snapshot_manager/main.py github-matrix \
+          --strategy ${{ inputs.strategy }} \
+        )" >> $GITHUB_OUTPUT
 
   build-on-copr:
     if: github.repository_owner == 'fedora-llvm-team'
@@ -41,97 +53,74 @@ jobs:
     runs-on: ubuntu-latest
     container: fedora:41
     steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/prepare-python
+
       - name: Setup Copr config file
         env:
           # You need to have those secrets in your repo.
           # See also: https://copr.fedorainfracloud.org/api/.
-          COPR_CONFIG_FILE: ${{ secrets.COPR_CONFIG }}
+          COPR_CONFIG: ${{ secrets.COPR_CONFIG }}
         run: |
           mkdir -p ~/.config
-          echo "$COPR_CONFIG_FILE" > ~/.config/copr
+          echo "$COPR_CONFIG" > ~/.config/copr
 
       - name: Install Copr CLI and required tools
         run: |
           dnf install -y copr-cli make bzip2 rpm-build pcre2-tools jq
 
-      - uses: actions/checkout@v4
-
-      - name: "Variables and functions"
-        shell: bash -e {0}
+      - name: "Variables"
         run: |
-          source scripts/functions.sh
-          [[ -n "${{ matrix.extra_script_file }}" ]] && source ${{ matrix.extra_script_file }}
-
           today=$(date +%Y%m%d)
           yesterday=$(date -d "${today} -1 day" +%Y%m%d)
 
-          chroots="$(get_chroots)"
-
           username=@fedora-llvm-team
           {
             echo "username=$username"
-            echo "chroots=$chroots"
-            echo "all_chroots=$all_chroots"
+            echo "yesterday=$yesterday"
             echo "project_today=${{ matrix.copr_ownername }}/${{ matrix.copr_project_tpl }}" | sed "s/YYYYMMDD/$today/"
             echo "project_yesterday=${{ matrix.copr_ownername }}/${{ matrix.copr_project_tpl }}" | sed "s/YYYYMMDD/$yesterday/"
             echo "project_target=${{ matrix.copr_target_project }}"
           } >> "$GITHUB_ENV"
 
       - name: "Check for Copr projects existence (yesterday, today, target)"
-        shell: bash -e {0}
         run: |
-          source scripts/functions.sh
-          [[ -n "${{ matrix.extra_script_file }}" ]] && source ${{ matrix.extra_script_file }}
+          # Checks if a copr project exists
+          function copr_project_exists(){
+            local project="$1";
+            copr get "$project" > /dev/null 2>&1
+          }
+
+          # Check general availability of projects
+          copr_project_exists ${{ env.project_today }} && todays_project_exists=yes
+          copr_project_exists ${{ env.project_yesterday }} && yesterdays_project_exists=yes
+          copr_project_exists ${{ env.project_target }} && target_project_exists=yes
 
           # Check if yesterday's project exists and all builds succeeded
-          yesterdays_project_exists=$(project_exists ${{ env.project_yesterday }})
-          if [[ "$yesterdays_project_exists" == "true" ]]; then
-            if ! has_all_good_builds ${{env.project_yesterday}}; then
-              yesterdays_project_exists=false
+          if [[ "$yesterdays_project_exists" == "yes" ]]; then
+            if ! python3 snapshot_manager/main.py has-all-good-builds --strategy ${{matrix.name}} --yyyymmdd ${{env.yesterday}}; then
+              yesterdays_project_exists=no
             fi
           fi
 
           {
-            echo "todays_project_exists=$(project_exists ${{ env.project_today }})"
+            echo "todays_project_exists=$todays_project_exists"
             echo "yesterdays_project_exists=$yesterdays_project_exists"
-            echo "target_project_exists=$(project_exists ${{ env.project_target }})"
+            echo "target_project_exists=$target_project_exists"
           } >> "$GITHUB_ENV"
 
-      - name: "Canceling active builds (if any) in today's Copr project before recreating it: ${{ env.project_today }}"
-        if: ${{ env.todays_project_exists == 'true' }}
-        shell: bash -e {0}
-        run: |
-          source scripts/functions.sh
-          [[ -n "${{ matrix.extra_script_file }}" ]] && source ${{ matrix.extra_script_file }}
-
-          build_ids=""
-          for build_id in $(get_active_build_ids ${{ env.project_today }}); do
-            echo "Canceling build with ID $build_id"
-            copr cancel "$build_id"
-            build_ids="$build_ids $build_id"
-          done
-          if [[ "$build_ids" != "" ]]; then
-            echo "Waiting for build IDs to be canceled: $build_ids"
-            copr watch-build "$build_ids" || true
-          fi
-
-      - name: "Deleting today's Copr project before recreating it: ${{ env.project_today }}"
-        if: ${{ env.todays_project_exists == 'true' }}
-        shell: bash -e {0}
+      - name: "Canceling active builds (if any) in today's Copr project and deleting it: ${{ env.project_today }}"
+        if: ${{ env.todays_project_exists == 'yes' }}
         run: |
-          source scripts/functions.sh
-          [[ -n "${{ matrix.extra_script_file }}" ]] && source ${{ matrix.extra_script_file }}
-
-          copr delete ${{ env.project_today }}
+          python3 snapshot_manager/main.py \
+            delete-project \
+            --strategy ${{matrix.name}} \
+            --yyyymmdd "$yyyymmdd"
 
       - name: "Create today's Copr project: ${{ env.project_today }}"
-        shell: bash -e {0}
         run: |
-          source scripts/functions.sh
-          [[ -n "${{ matrix.extra_script_file }}" ]] && source ${{ matrix.extra_script_file }}
-
           # shellcheck disable=SC2207
-          chroot_opts=($(for c in ${{ env.chroots }}; do echo -n " --chroot $c "; done))
+          chroot_opts=($(for c in ${{ matrix.chroots }}; do echo -n " --chroot $c "; done))
 
           copr create \
             --instructions "$(cat project-instructions.md)" \
@@ -146,12 +135,8 @@ jobs:
             "${chroot_opts[@]}" "${{ env.project_today }}"
 
       - name: "Enable snapshot_build build condition for all and swig:4.0 module in RHEL 8 build chroots (if any)"
-        shell: bash -e {0}
         run: |
-          source scripts/functions.sh
-          [[ -n "${{ matrix.extra_script_file }}" ]] && source ${{ matrix.extra_script_file }}
-
-          for chroot in ${{ env.chroots }}; do
+          for chroot in ${{ matrix.chroots }}; do
             copr edit-chroot --rpmbuild-with "snapshot_build" "${{ env.project_today }}/$chroot"
             if [[ "$chroot" == rhel-8-* ]]; then
               copr edit-chroot --modules "swig:4.0" "${{ env.project_today }}/$chroot"
@@ -162,11 +147,7 @@ jobs:
           done
 
       - name: "Create today's package"
-        shell: bash -e {0}
         run: |
-          source scripts/functions.sh
-          [[ -n "${{ matrix.extra_script_file }}" ]] && source ${{ matrix.extra_script_file }}
-
           copr add-package-scm \
             --clone-url ${{ matrix.clone_url }} \
             --commit ${{ matrix.clone_ref }} \
@@ -177,12 +158,8 @@ jobs:
             "${{ env.project_today }}"
 
       - name: "Build llvm package"
-        shell: bash -e {0}
         run: |
-          source scripts/functions.sh
-          [[ -n "${{ matrix.extra_script_file }}" ]] && source ${{ matrix.extra_script_file }}
-
-          for chroot in ${{ env.chroots }}; do
+          for chroot in ${{ matrix.chroots }}; do
             copr build-package \
               --timeout $((30*3600)) \
               --nowait \
@@ -193,22 +170,22 @@ jobs:
           done
 
       - name: "Delete target Copr project at ${{ env.project_target }} before forking to it"
-        if: ${{ env.yesterdays_project_exists == 'true' && env.target_project_exists == 'true' }}
+        if: ${{ env.yesterdays_project_exists == 'yes' && env.target_project_exists == 'yes' }}
         run: |
           copr delete "${{ env.project_target }}"
           # Give Copr some time to process the deletion, to avoid race conditions with forking.
           # TODO: Keep and eye on https://github.com/fedora-copr/copr/issues/2698 if there's a better way to handle this.
           sleep 1m
 
       - name: "Fork Copr project from ${{ env.project_yesterday }} to ${{ env.project_target }}"
-        if: ${{ env.yesterdays_project_exists == 'true' }}
+        if: ${{ env.yesterdays_project_exists == 'yes' }}
         run: |
           copr fork --confirm ${{ env.project_yesterday }} ${{ env.project_target }}
           copr modify --delete-after-days -1 --unlisted-on-hp off ${{ env.project_target }}
 
       - name: "Regenerate repos for target project ${{ env.project_target }}"
         # If yesterday's project didn't exist, we haven't forked and so we don't
         # need to regenerate the repos.
-        if: ${{ env.yesterdays_project_exists == 'true' }}
+        if: ${{ env.yesterdays_project_exists == 'yes' }}
         run: |
           copr regenerate-repos ${{ env.project_target }}

.github/workflows/retest-chroots-on-testing-farm.yml

@@ -84,7 +84,6 @@ jobs:
         run: |
           python3 snapshot_manager/main.py \
             --github-repo "${GITHUB_REPOSITORY}" \
-            --github-token-env GITHUB_TOKEN \
             retest \
             --trigger-comment-id ${{ github.event.comment.id }} \
             --issue-number ${{ github.event.issue.number }} \