Upgrade pgvector and improve eval workflow #41
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Evaluate | |
| on: | |
| workflow_dispatch: | |
| issue_comment: | |
| types: [created] | |
| # Set up permissions for deploying with secretless Azure federated credentials | |
| # https://learn.microsoft.com/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| evaluate: | |
| if: github.event_name == 'workflow_dispatch' || contains(github.event.comment.body, '#evaluate') | |
| runs-on: ubuntu-latest | |
| env: | |
| AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} | |
| AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} | |
| AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} | |
| AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} | |
| AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} | |
| steps: | |
| - name: Check for evaluate hash tag | |
| if: contains(github.event.comment.body, '#evaluate') | |
| run: | | |
| echo "Comment contains #evaluate hashtag" | |
| - uses: actions/checkout@v4 | |
| - name: Install PostgreSQL development libraries | |
| run: | | |
| sudo apt update | |
| sudo apt install postgresql-server-dev-14 | |
| - name: Setup postgres | |
| uses: ikalnytskyi/action-setup-postgres@v6 | |
| with: | |
| username: admin | |
| password: postgres | |
| database: postgres | |
| - name: Install pgvector on MacOS/Linux using install-pgvector.sh | |
| run: .github/workflows/install-pgvector.sh | |
| - name: Install python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install azd | |
| uses: Azure/[email protected] | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install -r requirements-dev.txt | |
| - name: Install app as editable app | |
| run: | | |
| python -m pip install -e src/backend | |
| - name: Setup local database with seed data | |
| run: | | |
| python ./src/backend/fastapi_app/setup_postgres_database.py | |
| python ./src/backend/fastapi_app/setup_postgres_seeddata.py | |
| env: | |
| POSTGRES_HOST: localhost | |
| POSTGRES_USERNAME: admin | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DATABASE: postgres | |
| POSTGRES_SSL: disable | |
| - name: Setup node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18 | |
| - name: Build frontend | |
| run: | | |
| cd ./src/frontend | |
| npm install | |
| npm run build | |
| - name: Install python packages | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements-dev.txt | |
| - name: Login to Azure | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ env.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ env.AZURE_TENANT_ID }} | |
| subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }} | |
| - name: Set az account | |
| uses: azure/CLI@v2 | |
| with: | |
| inlineScript: | | |
| az account set --subscription ${{env.AZURE_SUBSCRIPTION_ID}} | |
| - name: Log in with Azure (Federated Credentials) | |
| if: ${{ env.AZURE_CLIENT_ID != '' }} | |
| run: | | |
| azd auth login ` | |
| --client-id "$Env:AZURE_CLIENT_ID" ` | |
| --federated-credential-provider "github" ` | |
| --tenant-id "$Env:AZURE_TENANT_ID" | |
| shell: pwsh | |
| - name: Provision Infrastructure | |
| run: azd provision --no-prompt | |
| env: | |
| AZD_INITIAL_ENVIRONMENT_CONFIG: ${{ secrets.AZD_INITIAL_ENVIRONMENT_CONFIG }} | |
| - name: Run local server in background | |
| run: | | |
| RUNNER_TRACKING_ID="" && (nohup python3 -m uvicorn fastapi_app:create_app --factory > serverlogs.out 2> serverlogs.err &) | |
| env: | |
| OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }} | |
| OPENAI_EMBED_HOST: ${{ vars.OPENAI_EMBED_HOST }} | |
| AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }} | |
| AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }} | |
| AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }} | |
| AZURE_OPENAI_EMBED_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMBED_DEPLOYMENT }} | |
| AZURE_OPENAI_EMBED_MODEL: ${{ vars.AZURE_OPENAI_EMBED_MODEL }} | |
| AZURE_OPENAI_EMBED_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMBED_DIMENSIONS }} | |
| AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }} | |
| POSTGRES_HOST: localhost | |
| POSTGRES_USERNAME: admin | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DATABASE: postgres | |
| POSTGRES_SSL: disable | |
| - name: Evaluate local RAG flow | |
| run: | | |
| python evals/evaluate.py | |
| env: | |
| OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }} | |
| AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }} | |
| AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }} | |
| AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }} | |
| AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }} | |
| AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }} | |
| - name: Upload server logs as build artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: server_logs | |
| path: ./serverlogs.out | |
| - name: Upload server error logs as build artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: server_error_logs | |
| path: ./serverlogs.err | |
| - name: Upload eval results as build artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: eval_result | |
| path: ./src/api/evaluate/eval_results.jsonl | |
| - name: GitHub Summary Step | |
| if: ${{ success() }} | |
| working-directory: ./src/api | |
| run: | | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "📊 Promptflow Evaluation Results" >> $GITHUB_STEP_SUMMARY | |
| cat evaluate/eval_results.md >> $GITHUB_STEP_SUMMARY |