Upgrade pgvector and improve eval workflow #32
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Evaluate | |
| on: | |
| workflow_dispatch: | |
| issue_comment: | |
| types: [created] | |
| pull_request: | |
| branches: [ main ] | |
| # Set up permissions for deploying with secretless Azure federated credentials | |
| # https://learn.microsoft.com/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| evaluate: | |
| #if: github.event_name == 'workflow_dispatch' || contains(github.event.comment.body, '#evaluate') | |
| runs-on: ubuntu-latest | |
| env: | |
| AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} | |
| AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} | |
| AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} | |
| AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} | |
| AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} | |
| POSTGRES_HOST: localhost | |
| POSTGRES_USERNAME: postgres | |
| POSTGRES_PASSWORD: root | |
| POSTGRES_DATABASE: postgres | |
| POSTGRES_SSL: disable | |
| OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }} | |
| OPENAI_EMBED_HOST: ${{ vars.OPENAI_EMBED_HOST }} | |
| AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }} | |
| AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }} | |
| AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }} | |
| AZURE_OPENAI_EMBED_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMBED_DEPLOYMENT }} | |
| AZURE_OPENAI_EMBED_MODEL: ${{ vars.AZURE_OPENAI_EMBED_MODEL }} | |
| AZURE_OPENAI_EMBED_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMBED_DIMENSIONS }} | |
| AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }} | |
| AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }} | |
| AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }} | |
| steps: | |
| - name: Check for evaluate hash tag | |
| if: contains(github.event.comment.body, '#evaluate') | |
| run: | | |
| echo "Comment contains #evaluate hashtag" | |
| - uses: actions/checkout@v4 | |
| - name: Install pgvector | |
| run: | | |
| sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y | |
| sudo apt-get install postgresql-14-pgvector | |
| - name: Start postgres | |
| run: sudo systemctl start postgresql | |
| - name: Set password for postgres user | |
| run: sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'root'" | |
| - name: Create vector extension | |
| run: sudo -u postgres psql -c 'CREATE EXTENSION vector' | |
| - name: Install python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install azd | |
| uses: Azure/[email protected] | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install -r requirements-dev.txt | |
| - name: Install app as editable app | |
| run: | | |
| python -m pip install -e src/backend | |
| - name: Setup local database with seed data | |
| run: | | |
| python ./src/backend/fastapi_app/setup_postgres_database.py | |
| python ./src/backend/fastapi_app/setup_postgres_seeddata.py | |
| - name: Setup node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18 | |
| - name: Build frontend | |
| run: | | |
| cd ./src/frontend | |
| npm install | |
| npm run build | |
| - name: Install python packages | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements-dev.txt | |
| - name: Login to Azure | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ env.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ env.AZURE_TENANT_ID }} | |
| subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }} | |
| - name: Set az account | |
| uses: azure/CLI@v2 | |
| with: | |
| inlineScript: | | |
| az account set --subscription ${{env.AZURE_SUBSCRIPTION_ID}} | |
| - name: Log in with Azure (Federated Credentials) | |
| if: ${{ env.AZURE_CLIENT_ID != '' }} | |
| run: | | |
| azd auth login ` | |
| --client-id "$Env:AZURE_CLIENT_ID" ` | |
| --federated-credential-provider "github" ` | |
| --tenant-id "$Env:AZURE_TENANT_ID" | |
| shell: pwsh | |
| - name: Run local server in background | |
| run: | | |
| RUNNER_TRACKING_ID="" && (nohup python3 -m uvicorn fastapi_app:create_app --factory > serverlogs.out 2> serverlogs.err &) | |
| - name: Evaluate local RAG flow | |
| run: | | |
| python evals/evaluate.py | |
| - name: Upload server logs as build artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: server_logs | |
| path: ./serverlogs.out | |
| - name: Upload server error logs as build artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: server_error_logs | |
| path: ./serverlogs.err | |
| - name: Upload eval results as build artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: eval_result | |
| path: ./src/api/evaluate/eval_results.jsonl | |
| - name: GitHub Summary Step | |
| if: ${{ success() }} | |
| working-directory: ./src/api | |
| run: | | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "📊 Promptflow Evaluation Results" >> $GITHUB_STEP_SUMMARY | |
| cat evaluate/eval_results.md >> $GITHUB_STEP_SUMMARY |