CI test

waynehamadi · waynehamadi · commit cdb489636b03 · 2023-06-06T12:05:46.000-07:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -148,22 +148,38 @@ jobs:
 
       - name: Run pytest with coverage
         run: |
-          pytest -n auto --cov=autogpt --cov-branch --cov-report term-missing --cov-report xml \
-            tests/unit tests/integration tests/challenges
-          python tests/challenges/utils/build_current_score.py
+          echo "test" >> tests/Auto-GPT-test-cassettes/test_write_file/test_write_file.yaml
+          echo "test" >> tests/challenges/current_score.json
         env:
           CI: true
           PROXY: ${{ secrets.PROXY }}
           AGENT_MODE: ${{ vars.AGENT_MODE }}
           AGENT_TYPE: ${{ vars.AGENT_TYPE }}
 
+      - id: get_cassettes_diff
+        name: Identify differences between cassettes of current PR and target branch
+        if: ${{ startsWith(github.event_name, 'pull_request') }}
+        run: |
+          cd tests/Auto-GPT-test-cassettes
+          cassette_diff=$(git diff --name-only --diff-filter=M -- '**/*.yaml')
+          
+          if [ -n "$cassette_diff" ]; then
+          echo "updated=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Attempt to beat Challenges if cassettes changed
+        if: ${{ startsWith(github.event_name, 'pull_request') }} && "${{ steps.get_cassettes_diff.outputs.updated }}" == "true"
+        run: |
+          echo "test" >> tests/Auto-GPT-test-cassettes/test_write_file/test_write_file.yaml
+          echo "test" >> tests/challenges/current_score.json
+
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
 
       - id: setup_git_auth
         name: Set up git token authentication
-        # Cassettes may be pushed even when tests fail
-        if: success() || failure()
+        # Cassettes may be pushed even when tests fail.
+        if: always() && (startsWith(github.event_name, 'pull_request') || success())
         run: |
           config_key="http.${{ github.server_url }}/.extraheader"
           base64_pat=$(echo -n "pat:${{ secrets.PAT_REVIEW }}" | base64 -w0)
@@ -177,23 +193,31 @@ jobs:
 
           echo "config_key=$config_key" >> $GITHUB_OUTPUT
 
-      - name: Push updated challenge scores
-        if: github.event_name == 'push'
+      - id: push_challenge_scores
+        name: Push updated challenge scores
         run: |
           score_file="tests/challenges/current_score.json"
 
           if ! git diff --quiet $score_file; then
             git add $score_file
             git commit -m "Update challenge scores"
-            git push origin HEAD:${{ github.ref_name }}
+            if [ "${{ startsWith(github.event_name, 'pull_request') }}" = "true" ]; then
+              updated_score_branch="update-score-${{ github.event.pull_request.number }}"
+              git checkout -b $updated_score_branch
+              git remote add base_repo https://github.com/${{ github.event.pull_request.base.repo.owner.login }}/${{ github.event.pull_request.base.repo.name }}.git
+              git push -f base_repo $updated_score_branch
+              echo "updated_score_branch=$updated_score_branch" >> $GITHUB_OUTPUT
+            else
+              git push origin HEAD:${{ github.ref }}
+            fi
           else
             echo "The challenge scores didn't change."
           fi
 
       - id: push_cassettes
         name: Push updated cassettes
         # For pull requests, push updated cassettes even when tests fail
-        if: github.event_name == 'push' || success() || failure()
+        if: always() && (startsWith(github.event_name, 'pull_request') || success())
         run: |
           if [ "${{ startsWith(github.event_name, 'pull_request') }}" = "true" ]; then
             is_pull_request=true
@@ -214,9 +238,7 @@ jobs:
               git commit -m "Update cassette submodule"
               git push origin HEAD:$cassette_branch
             fi
-            echo "updated=true" >> $GITHUB_OUTPUT
           else
-            echo "updated=false" >> $GITHUB_OUTPUT
             echo "No cassette changes to commit"
           fi
 
@@ -225,22 +247,59 @@ jobs:
         run: |
           git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}'
           git submodule foreach git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}'
+                    
+          echo ${{ secrets.PAT_REVIEW }} | gh auth login --with-token
+
+      - id: create_pr
+        name: Create Pull Request if score changed, if not cleanup branch and PR
+        if: ${{ startsWith(github.event_name, 'pull_request')}}
+        run: |
+          base_repo_owner=${{ github.event.pull_request.base.repo.owner.login }}
+          base_repo_name=${{ github.event.pull_request.base.repo.name }}
+          updated_score_branch=${{steps.push_challenge_scores.outputs.updated_score_branch}}
+          pr_number=$(gh api repos/$base_repo_owner/$base_repo_name/pulls -q ".[] | select(.head.ref == \"$updated_score_branch\") | .number")
+          if [ -z "$updated_score_branch" ]; then
+            if [ -n "$pr_number" ]; then
+              gh api -X PATCH repos/$base_repo_owner/$base_repo_name/pulls/$pr_number -f state=closed
+            fi
+            # Delete branch
+            gh api -X DELETE repos/$base_repo_owner/$base_repo_name/git/refs/heads/$updated_score_branch || echo "Branch not found or already deleted."
+            echo "Score didn't change."
+            exit 0
+          fi
+          if [ -z "$pr_number" ] || [ "$(gh api repos/$base_repo_owner/$base_repo_name/pulls/$pr_number --jq ".state")" = "closed" ]; then
+            pr_url=$(gh pr create --title "Update Score In Pull Request Number ${{ github.event.pull_request.number }}" \
+              --head "${{ github.event.pull_request.base.repo.owner.login }}:$updated_score_branch" \
+              --base "${{ github.event.pull_request.head.ref }}" \
+              --body "This pull request updates the current score of Auto-GPT. Please check the files changed and merge the pull request." \
+              --repo ${{ github.event.pull_request.head.repo.full_name }})
+          else
+            pr_url=$(gh pr view $pr_number --json url --jq '.url')
+          fi
+          echo "pr_url=$pr_url" >> $GITHUB_OUTPUT  
 
       - name: Apply "behaviour change" label and comment on PR
         if: ${{ startsWith(github.event_name, 'pull_request') }}
         run: |
-          PR_NUMBER=${{ github.event.pull_request.number }}
-          TOKEN=${{ secrets.PAT_REVIEW }}
           REPO=${{ github.repository }}
-
-          if [[ "${{ steps.push_cassettes.outputs.updated }}" == "true" ]]; then
+          COMMENT_BODY=""
+          SCORE_CHANGED_MESSAGE="The pipeline will fail until you merge the updated score: ${{ steps.create_pr.outputs.pr_url }}"
+          if [[ "${{ steps.get_cassettes_diff.outputs.updated }}" == "true" ]]; then
             echo "Adding label and comment..."
             curl -X POST \
-            -H "Authorization: Bearer $TOKEN" \
+            -H "Authorization: Bearer ${{ secrets.PAT_REVIEW }}" \
             -H "Accept: application/vnd.github.v3+json" \
-            https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels \
+            https://api.github.com/repos/$REPO/issues/${{ github.event.pull_request.number }}/labels \
             -d '{"labels":["behaviour change"]}'
-
-            echo $TOKEN | gh auth login --with-token
-            gh api repos/$REPO/issues/$PR_NUMBER/comments -X POST -F body="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged."
+            COMMENT_BODY="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged. "
+          fi
+          if [[ -n "${{ steps.create_pr.outputs.pr_url }}" ]]; then
+            COMMENT_BODY="$COMMENT_BODY$SCORE_CHANGED_MESSAGE"
+          fi
+          if [ ! -z "$COMMENT_BODY" ]; then
+            gh api repos/$REPO/issues/${{ github.event.pull_request.number }}/comments -X POST -F body="$COMMENT_BODY"
+          fi
+          if [[ -n "${{ steps.create_pr.outputs.pr_url }}" ]]; then
+            echo "$SCORE_CHANGED_MESSAGE"
+            exit 1
           fi
diff --git a/.gitmodules b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "tests/Auto-GPT-test-cassettes"]
     path = tests/Auto-GPT-test-cassettes
     url = https://github.com/Significant-Gravitas/Auto-GPT-test-cassettes
-    branch = master
+    branch = ci-test-2
diff --git a/tests/Auto-GPT-test-cassettes b/tests/Auto-GPT-test-cassettes
@@ -1 +1 @@
-Subproject commit 38ecb0145aa3a88e2eb5f04a556146614a2882e1
+Subproject commit b5d1e20afdd18de8097b8d9bf89746bc0ac7fc07
