[issue-7848] [SDK] fix: fail closed on ambiguous LLM-judge JSON output #5943
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # The workflow to run tests for third-party libraries integrations that require OPIK server running | |
| # | |
| name: SDK E2E Libraries Integration Tests | |
| permissions: | |
| contents: read | |
| packages: read | |
| run-name: "SDK E2E Libraries Integration Tests ${{ github.ref_name }} by @${{ github.actor }}" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| libs: | |
| description: "Choose specific library to test against or all" | |
| required: true | |
| type: choice | |
| options: | |
| - all | |
| - adk | |
| - litellm | |
| - opentelemetry | |
| - pydantic_ai | |
| schedule: | |
| - cron: "0 0 */2 * *" | |
| pull_request: | |
| paths: | |
| - 'sdks/python/**' | |
| - '.github/workflows/sdk-e2e-library-tests.yaml' | |
| push: | |
| branches: | |
| - 'main' | |
| paths: | |
| - 'sdks/python/**' | |
| - '.github/workflows/sdk-e2e-library-tests.yaml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| env: | |
| OPIK_ENABLE_LITELLM_MODELS_MONITORING: True # unlike other workflows, here we need it for tests | |
| OPIK_SENTRY_ENABLE: False | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| LIBS: ${{ github.event.inputs.libs != '' && github.event.inputs.libs || 'all' }} | |
| OPIK_URL_OVERRIDE: http://localhost:8080 | |
| jobs: | |
| has_needed_secrets: | |
| name: Check Secrets | |
| runs-on: ubuntu-latest | |
| outputs: | |
| has_secrets: ${{ steps.init.outputs.has_secrets }} | |
| steps: | |
| - name: Print has secrets into output | |
| id: init | |
| run: | | |
| echo "has_secrets=${{ secrets.OPENAI_API_KEY != '' }}" >> "$GITHUB_OUTPUT" | |
| missed_api_key_warning: | |
| name: Missed OpenAI API Key Warning | |
| needs: [has_needed_secrets] | |
| runs-on: ubuntu-latest | |
| if: ${{ needs.has_needed_secrets.outputs.has_secrets == 'false' }} | |
| steps: | |
| - name: Print disabled message | |
| run: | | |
| echo "::warning::SDK E2E Libraries Integration Tests workflow is disabled because OPENAI_API_KEY is not set" | |
| build-opik: | |
| needs: [has_needed_secrets] | |
| if: ${{ needs.has_needed_secrets.outputs.has_secrets == 'true' }} | |
| # The reusable build workflow pushes images to GHCR; pass down the | |
| # packages: write grant here rather than at the workflow level (a called | |
| # workflow's permissions can only be maintained or reduced from the | |
| # caller's, never elevated). | |
| permissions: | |
| contents: read | |
| packages: write | |
| uses: ./.github/workflows/build_e2e_docker.yaml | |
| run_e2e_lib_integration: | |
| name: E2E Lib Integration Python ${{matrix.python_version}} | |
| needs: [has_needed_secrets, build-opik] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| if: ${{ needs.has_needed_secrets.outputs.has_secrets == 'true' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python_version: ${{ fromJSON(vars.PYTHON_VERSIONS) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Login to GHCR | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pull Docker images from GHCR | |
| env: | |
| TAG: ${{ needs.build-opik.outputs.image_tag }} | |
| run: | | |
| docker pull "ghcr.io/comet-ml/opik/opik-backend:$TAG" | |
| docker pull "ghcr.io/comet-ml/opik/opik-python-backend:$TAG" | |
| docker tag "ghcr.io/comet-ml/opik/opik-backend:$TAG" ghcr.io/comet-ml/opik/opik-backend:latest | |
| docker tag "ghcr.io/comet-ml/opik/opik-python-backend:$TAG" ghcr.io/comet-ml/opik/opik-python-backend:latest | |
| - name: Install uv and set the Python version ${{ matrix.python_version }} | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| python-version: ${{ matrix.python_version }} | |
| - name: Make LIBS variable global (workaround for cron) | |
| id: init | |
| run: | | |
| echo "LIBS=${{ env.LIBS }}" >> "$GITHUB_OUTPUT" | |
| - name: Run latest Opik server | |
| env: | |
| OPIK_USAGE_REPORT_ENABLED: false | |
| # Avoid Buildx Bake concurrent image export collisions in CI. | |
| COMPOSE_BAKE: false | |
| OPIK_BACKEND_PULL_POLICY: never | |
| PYTHON_BACKEND_PULL_POLICY: never | |
| shell: bash | |
| run: | | |
| cd ${{ github.workspace }} | |
| ./opik.sh --backend --port-mapping | |
| - name: Check Opik server availability | |
| shell: bash | |
| run: | | |
| chmod +x ${{ github.workspace }}/tests_end_to_end/installer_utils/*.sh | |
| cd ${{ github.workspace }}/deployment/docker-compose | |
| echo "Check Docker pods are up" | |
| ${{ github.workspace }}/tests_end_to_end/installer_utils/check_docker_compose_pods.sh | |
| echo "Check backend health" | |
| ${{ github.workspace }}/tests_end_to_end/installer_utils/check_backend.sh | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v3 | |
| with: | |
| credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}' | |
| - name: ADK Tests Python ${{ matrix.python_version }} | |
| if: contains(fromJSON('["adk", "all"]'), env.LIBS) | |
| env: | |
| GOOGLE_CLOUD_LOCATION: global | |
| GOOGLE_CLOUD_PROJECT: opik-sdk-tests | |
| GOOGLE_GENAI_USE_VERTEXAI: TRUE | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENAI_ORG_ID: ${{ secrets.OPENAI_ORG_ID }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| uses: ./.github/actions/install_opik_and_run_e2e_lib_integration_tests | |
| with: | |
| python_version: ${{ matrix.python_version }} | |
| library_name: adk | |
| # This workflow is used to test the external LiteLLM integration that lives in litellm repository. | |
| # However, there is another, internal, integration with LiteLLM that lives in opik.integrations.litellm, it's being tested | |
| # in a separate workflow: .github/workflows/lib-litellm-tests.yml | |
| - name: LiteLLM Tests Python ${{ matrix.python_version }} | |
| if: contains(fromJSON('["litellm", "all"]'), env.LIBS) | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENAI_ORG_ID: ${{ secrets.OPENAI_ORG_ID }} | |
| uses: ./.github/actions/install_opik_and_run_e2e_lib_integration_tests | |
| with: | |
| python_version: ${{ matrix.python_version }} | |
| library_name: litellm | |
| - name: OpenTelemetry Tests Python ${{ matrix.python_version }} | |
| if: contains(fromJSON('["opentelemetry", "all"]'), env.LIBS) | |
| uses: ./.github/actions/install_opik_and_run_e2e_lib_integration_tests | |
| with: | |
| python_version: ${{ matrix.python_version }} | |
| library_name: opentelemetry | |
| - name: PydanticAI Tests Python ${{ matrix.python_version }} | |
| if: contains(fromJSON('["pydantic_ai", "all"]'), env.LIBS) | |
| uses: ./.github/actions/install_opik_and_run_e2e_lib_integration_tests | |
| with: | |
| python_version: ${{ matrix.python_version }} | |
| library_name: pydantic_ai | |
| - name: Stop opik server | |
| run: | | |
| cd ${{ github.workspace }} | |
| ./opik.sh --stop |