Create cassette submodule (Significant-Gravitas#4420)

* gfeat: specify directory of cassettes and automatically load them depending on module fix: formatting for linter test: commit newly generated cassettes to their respective folder tests: update latest fixtures with master fix: update .gitattributes with updated path to cassettes fix: use cassettes from master instead of generating them myself fix: update path in .gitattributes fix: make sure to match default functionality by using test name for cassette directory fix: actually add git submodule ci: checkout git submodules in CI ci: update git submodules separately to ensure it gets called feat: add a hooks directory so we can update git submodules on post-checkout feat: make sure we push the tests/cassettes submodule on merge into master ci: remove unused code now that we are using git submodules to keep cassettes in sync fix: simplify how we load the submodule and fix updating cassettes on merge to master chore: remove echo of checkout hook, it's unneeded ci: remove unneccesary step * cassettes submodule * cassettes submodule * cassettes submodule * cassettes submodule * cassettes submodule --------- Co-authored-by: Stefan Ayala <[email protected]>
PointerSoftware · May 26, 2023 · e7c0d33 · e7c0d33
1 parent 6c45fcd
commit e7c0d33
Show file tree

Hide file tree

Showing 18 changed files with 171 additions and 14,404 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,5 +1,5 @@
 # Exclude VCR cassettes from stats
-tests/**/cassettes/**.y*ml linguist-generated
+tests/Auto-GPT-test-cassettes/**/**.y*ml linguist-generated
 
 # Mark documentation as such
 docs/**.md linguist-documentation
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -2,9 +2,11 @@ name: Python CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, ci-test*]
+    paths-ignore:
+      - 'tests/Auto-GPT-test-cassettes'
   pull_request_target:
-    branches: [ master, stable ]
+    branches: [ master, stable , ci-test*]
 
 concurrency:
   group: ${{ format('ci-{0}', github.head_ref && format('pr-{0}', github.event.pull_request.number) || github.sha) }}
@@ -71,6 +73,33 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
+          submodules: true
+          token: ${{ secrets.PAT_REVIEW }}
+
+      - name: Use cassettes previously sent
+        if: ${{ github.event_name == 'pull_request_target' }}
+        run: |
+          new_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}"
+          git config --global user.name "Github Actions"
+          git config --global user.email "[email protected]"
+          cd tests/Auto-GPT-test-cassettes
+
+          if git ls-remote --exit-code --heads origin $new_branch ; then
+            git fetch origin $new_branch:$new_branch
+            git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }}
+            
+            git checkout $new_branch
+
+            if git merge --no-commit --no-ff ${{ github.event.pull_request.base.ref }}; then
+              echo "No merge conflict detected. We can use the cassettes previously sent."
+            else
+              echo "Merge conflict detected. This means we cannot use the cassettes previously sent, so we will take the cassettes of ${{ github.event.pull_request.base.ref }}."
+              git merge --abort
+              git checkout ${{ github.event.pull_request.base.ref }}
+            fi
+          else
+            echo "Branch $new_branch does not exist. We will use the cassettes of ${{ github.event.pull_request.base.ref }}"
+          fi
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
@@ -82,7 +111,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
-      - name: Run unittest tests with coverage
+      - name: Run pytest tests with coverage
         run: |
           pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term
         env:
@@ -93,3 +122,82 @@ jobs:
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
+
+      - name: Update cassette submodule to push target if push event
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          cd tests/Auto-GPT-test-cassettes
+          current_branch=$(echo ${{ github.ref }} | sed -e "s/refs\/heads\///g")
+          git fetch origin $current_branch
+          git config --global user.name "Auto-GPT-Bot"
+          git config --global user.email "[email protected]"
+          git add .
+
+          # Check if there are any changes
+          if ! git diff-index --quiet HEAD; then
+              git commit -m "Auto-update cassettes after Push event"
+              git pull --rebase origin $current_branch
+
+              git push origin HEAD:refs/heads/$current_branch
+
+              cd ../..
+              git add tests/Auto-GPT-test-cassettes
+              git commit -m "Update submodule reference"
+              git push origin HEAD:refs/heads/$current_branch
+          else
+              echo "No changes to commit"
+              exit 0
+          fi
+
+      - name: Update cassette submodule to submodule branch if PR event
+        if: ${{ github.event_name == 'pull_request_target' }}
+        run: |
+          new_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}"
+
+          cd tests/Auto-GPT-test-cassettes
+          git config --global user.name "Auto-GPT-Bot"
+          git config --global user.email "[email protected]"
+          git add .
+
+          # Check if there are any changes
+          if ! git diff-index --quiet HEAD; then
+              git commit -m "Auto-update cassettes after merging PR #$pr_number"
+              git push -f origin HEAD:refs/heads/$new_branch
+              
+          else
+              echo "No changes to commit"
+              exit 0
+          fi
+          git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }} || echo "DIFF_EXISTS=false" >> $GITHUB_ENV
+          differences=$(git diff origin/$new_branch origin/${{ github.event.pull_request.base.ref }})
+    
+          if [ -n "$differences" ]; then
+            echo "DIFF_EXISTS=true" >> $GITHUB_ENV
+          else
+            echo "DIFF_EXISTS=false" >> $GITHUB_ENV
+          fi
+
+      - name: Apply or remove prompt change label and comment
+        if: ${{ github.event_name == 'pull_request_target' }}
+        run: |
+          PR_NUMBER=${{ github.event.pull_request.number }}
+          TOKEN=${{ secrets.PAT_REVIEW }}
+          REPO=${{ github.repository }}
+
+          if [[ "${{ env.DIFF_EXISTS }}" == "true" ]]; then
+            echo "Adding label and comment..."
+            curl -X POST \
+            -H "Authorization: Bearer $TOKEN" \
+            -H "Accept: application/vnd.github.v3+json" \
+            https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels \
+            -d '{"labels":["prompt change"]}'
+
+            echo $TOKEN | gh auth login --with-token
+            gh api repos/$REPO/issues/$PR_NUMBER/comments -X POST -F body="You changed AutoGPT's prompt. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged."
+          else
+            echo "Removing label..."
+            curl -X DELETE \
+            -H "Authorization: Bearer $TOKEN" \
+            -H "Accept: application/vnd.github.v3+json" \
+            https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels/prompt%20change
+          fi
diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml
@@ -76,41 +76,44 @@ jobs:
     timeout-minutes: 30
     needs: build
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-
-    - id: build
-      name: Build image
-      uses: docker/build-push-action@v3
-      with:
-        build-args: BUILD_TYPE=dev  # include pytest
-        tags: ${{ env.IMAGE_NAME }}
-        load: true                  # save to docker images
-        # cache layers in GitHub Actions cache to speed up builds
-        cache-from: type=gha,scope=docker-dev
-        cache-to: type=gha,scope=docker-dev,mode=max
-
-    - id: test
-      name: Run tests
-      env:
-        CI: true
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      run: |
-        set +e
-        test_output=$(
-          docker run --env CI --env OPENAI_API_KEY --entrypoint python ${{ env.IMAGE_NAME }} -m \
-          pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term 2>&1
-        )
-        test_failure=$?
-
-        echo "$test_output"
-
-        cat << $EOF >> $GITHUB_STEP_SUMMARY
-        # Tests $([ $test_failure = 0 ] && echo '✅' || echo '❌')
-        \`\`\`
-        $test_output
-        \`\`\`
-        $EOF
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - id: build
+        name: Build image
+        uses: docker/build-push-action@v3
+        with:
+          build-args: BUILD_TYPE=dev  # include pytest
+          tags: ${{ env.IMAGE_NAME }}
+          load: true                  # save to docker images
+          # cache layers in GitHub Actions cache to speed up builds
+          cache-from: type=gha,scope=docker-dev
+          cache-to: type=gha,scope=docker-dev,mode=max
+
+      - id: test
+        name: Run tests
+        env:
+          CI: true
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          set +e
+          test_output=$(
+            docker run --env CI --env OPENAI_API_KEY --entrypoint python ${{ env.IMAGE_NAME }} -m \
+            pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term 2>&1
+          )
+          test_failure=$?
+  
+          echo "$test_output"
+
+          cat << $EOF >> $GITHUB_STEP_SUMMARY
+          # Tests $([ $test_failure = 0 ] && echo '✅' || echo '❌')
+          \`\`\`
+          $test_output
+          \`\`\`
+          $EOF
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,4 @@
+[submodule "tests/Auto-GPT-test-cassettes"]
+    path = tests/Auto-GPT-test-cassettes
+    url = https://github.com/Significant-Gravitas/Auto-GPT-test-cassettes
+    branch = master
diff --git a/autogpt/llm/utils/__init__.py b/autogpt/llm/utils/__init__.py
@@ -96,7 +96,7 @@ def _wrapped(*args, **kwargs):
                         user_warned = True
 
                 except APIError as e:
-                    if (e.http_status != 502) or (attempt == num_attempts):
+                    if (e.http_status not in [502, 429]) or (attempt == num_attempts):
                         raise
 
                 backoff = backoff_base ** (attempt + 2)

diff --git a/hooks/post-checkout b/hooks/post-checkout
@@ -0,0 +1,2 @@
+#!/bin/sh
+git submodule update --init --remote --recursive
diff --git a/hooks/post-rewrite b/hooks/post-rewrite
@@ -0,0 +1,4 @@
+#!/bin/sh
+case "$1" in
+    rebase) git submodule update --init --recursive ;;
+esac
diff --git a/tests/Auto-GPT-test-cassettes b/tests/Auto-GPT-test-cassettes
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -13,6 +13,12 @@
 PROXY = os.environ.get("PROXY")
 
 
+@pytest.fixture()
+def vcr_cassette_dir(request):
+    test_name = os.path.splitext(request.node.name)[0]
+    return os.path.join("tests/Auto-GPT-test-cassettes", test_name)
+
+
 @pytest.fixture()
 def workspace_root(tmp_path: Path) -> Path:
     return tmp_path / "home/users/monty/auto_gpt_workspace"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		#!/bin/sh
		git submodule update --init --remote --recursive