Run Frontend Tests #1071
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Frontend Tests | |
| on: | |
| workflow_call: | |
| secrets: | |
| OPENAI_API_KEY: | |
| required: true | |
| STORE_API_KEY: | |
| required: true | |
| ANTHROPIC_API_KEY: | |
| required: true | |
| TAVILY_API_KEY: | |
| required: true | |
| inputs: | |
| suites: | |
| description: "Test suites to run (JSON array)" | |
| required: false | |
| type: string | |
| default: '[]' | |
| release: | |
| description: "Whether this is a release build" | |
| required: false | |
| type: boolean | |
| default: false | |
| tests_folder: | |
| description: "(Optional) Tests to run" | |
| required: false | |
| type: string | |
| default: "tests" | |
| ref: | |
| description: "(Optional) ref to checkout" | |
| required: false | |
| type: string | |
| workflow_dispatch: | |
| inputs: | |
| suites: | |
| description: "Test suites to run (JSON array)" | |
| required: false | |
| type: string | |
| default: '[]' | |
| release: | |
| description: "Whether this is a release build" | |
| required: false | |
| type: boolean | |
| default: false | |
| tests_folder: | |
| description: "(Optional) Tests to run" | |
| required: false | |
| type: string | |
| default: "tests" | |
| env: | |
| NODE_VERSION: "21" | |
| PYTHON_VERSION: "3.12" | |
| # Define the directory where Playwright browsers will be installed. | |
| # Adjust if your project uses a different path. | |
| PLAYWRIGHT_BROWSERS_PATH: "ms-playwright" | |
| jobs: | |
| determine-test-suite: | |
| name: Determine Test Suites and Shard Distribution | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.setup-matrix.outputs.matrix }} | |
| test_grep: ${{ steps.set-matrix.outputs.test_grep }} | |
| suites: ${{ steps.set-matrix.outputs.suites }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.ref || github.ref }} | |
| fetch-depth: 0 | |
| - name: Paths Filter | |
| id: filter | |
| uses: dorny/paths-filter@v3 | |
| with: | |
| filters: .github/changes-filter.yaml | |
| - name: Determine Test Suites from Changes | |
| id: set-matrix | |
| run: | | |
| # Start with input suites if provided, otherwise empty array | |
| echo "Changes filter output: ${{ steps.filter.outputs }}" | |
| SUITES='${{ inputs.suites }}' | |
| echo "Initial suites: $SUITES" | |
| TEST_GREP="" | |
| RELEASE="${{ inputs.release || 'false' }}" | |
| echo "Release build: $RELEASE" | |
| # Only set to release if it's explicitly a release build | |
| if [[ "$RELEASE" == "true" ]]; then | |
| SUITES='["release"]' | |
| echo "Release build detected - setting suites to: $SUITES" | |
| # grep pattern for release is the @release tag - run all tests | |
| TEST_GREP="--grep=\"@release\"" | |
| else | |
| # If input suites were not provided, determine based on changes | |
| if [[ "$SUITES" == "[]" ]]; then | |
| echo "No input suites provided - determining from changes" | |
| TAGS=() | |
| # Add suites and tags based on changed files | |
| if [[ "${{ steps.filter.outputs.components }}" == "true" ]]; then | |
| SUITES=$(echo $SUITES | jq -c '. += ["components"]') | |
| TAGS+=("@components") | |
| echo "Added components suite" | |
| fi | |
| if [[ "${{ steps.filter.outputs.starter-projects }}" == "true" ]]; then | |
| SUITES=$(echo $SUITES | jq -c '. += ["starter-projects"]') | |
| TAGS+=("@starter-projects") | |
| echo "Added starter-projects suite" | |
| fi | |
| if [[ "${{ steps.filter.outputs.workspace }}" == "true" ]]; then | |
| SUITES=$(echo $SUITES | jq -c '. += ["workspace"]') | |
| TAGS+=("@workspace") | |
| echo "Added workspace suite" | |
| fi | |
| if [[ "${{ steps.filter.outputs.api }}" == "true" ]]; then | |
| SUITES=$(echo $SUITES | jq -c '. += ["api"]') | |
| TAGS+=("@api") | |
| echo "Added api suite" | |
| fi | |
| if [[ "${{ steps.filter.outputs.database }}" == "true" ]]; then | |
| SUITES=$(echo $SUITES | jq -c '. += ["database"]') | |
| TAGS+=("@database") | |
| echo "Added database suite" | |
| fi | |
| # Create grep pattern if we have tags | |
| if [ ${#TAGS[@]} -gt 0 ]; then | |
| # Join tags with | for OR logic | |
| REGEX_PATTERN=$(IFS='|'; echo "${TAGS[*]}") | |
| TEST_GREP="--grep=\"${REGEX_PATTERN}\"" | |
| fi | |
| else | |
| # Process input suites to tags | |
| TAGS=() | |
| if echo "$SUITES" | jq -e 'contains(["components"])' > /dev/null; then | |
| TAGS+=("@components") | |
| fi | |
| if echo "$SUITES" | jq -e 'contains(["starter-projects"])' > /dev/null; then | |
| TAGS+=("@starter-projects") | |
| fi | |
| if echo "$SUITES" | jq -e 'contains(["workspace"])' > /dev/null; then | |
| TAGS+=("@workspace") | |
| fi | |
| if echo "$SUITES" | jq -e 'contains(["api"])' > /dev/null; then | |
| TAGS+=("@api") | |
| fi | |
| if echo "$SUITES" | jq -e 'contains(["database"])' > /dev/null; then | |
| TAGS+=("@database") | |
| fi | |
| if [ ${#TAGS[@]} -gt 0 ]; then | |
| # Join tags with | for OR logic | |
| REGEX_PATTERN=$(IFS='|'; echo "${TAGS[*]}") | |
| TEST_GREP="--grep \"${REGEX_PATTERN}\"" | |
| fi | |
| fi | |
| fi | |
| # Ensure compact JSON output | |
| SUITES=$(echo "$SUITES" | jq -c '.') | |
| echo "Final test suites to run: $SUITES" | |
| echo "Test grep pattern: $TEST_GREP" | |
| # Ensure proper JSON formatting for matrix output | |
| echo "matrix=$(echo $SUITES | jq -c .)" >> $GITHUB_OUTPUT | |
| echo "test_grep=$TEST_GREP" >> $GITHUB_OUTPUT | |
| - name: Setup Node ${{ env.NODE_VERSION }} | |
| uses: actions/setup-node@v4 | |
| id: setup-node | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: "npm" | |
| cache-dependency-path: ./src/frontend/package-lock.json | |
| - name: Install Frontend Dependencies | |
| run: npm ci | |
| working-directory: ./src/frontend | |
| - name: Calculate Test Shards Distribution | |
| id: setup-matrix | |
| run: | | |
| cd src/frontend | |
| # Get the test count using playwright's built-in grep | |
| if [ -n "${{ steps.set-matrix.outputs.test_grep }}" ]; then | |
| TEST_COUNT=$(npx playwright test ${{ inputs.tests_folder }} ${{ steps.set-matrix.outputs.test_grep }} --list | wc -l) | |
| else | |
| TEST_COUNT=$(npx playwright test ${{ inputs.tests_folder }} --list | wc -l) | |
| fi | |
| echo "Total tests to run: $TEST_COUNT" | |
| # Calculate optimal shard count - 1 shard per 5 tests, min 1, max 10 | |
| SHARD_COUNT=$(( (TEST_COUNT + 4) / 5 )) | |
| if [ $SHARD_COUNT -lt 1 ]; then | |
| SHARD_COUNT=1 | |
| elif [ $SHARD_COUNT -gt 10 ]; then | |
| SHARD_COUNT=10 | |
| fi | |
| # Create the matrix combinations string | |
| MATRIX_COMBINATIONS="" | |
| for i in $(seq 1 $SHARD_COUNT); do | |
| if [ $i -gt 1 ]; then | |
| MATRIX_COMBINATIONS="$MATRIX_COMBINATIONS," | |
| fi | |
| MATRIX_COMBINATIONS="$MATRIX_COMBINATIONS{\"shardIndex\": $i, \"shardTotal\": $SHARD_COUNT}" | |
| done | |
| echo "matrix={\"include\":[$MATRIX_COMBINATIONS]}" >> "$GITHUB_OUTPUT" | |
| setup-and-test: | |
| name: Playwright Tests - Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} | |
| runs-on: ubuntu-latest | |
| if: ${{ needs.determine-test-suite.outputs.test_grep != '' }} | |
| needs: determine-test-suite | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.determine-test-suite.outputs.matrix) }} | |
| env: | |
| OPENAI_API_KEY: ${{ inputs.openai_api_key || secrets.OPENAI_API_KEY }} | |
| STORE_API_KEY: ${{ inputs.store_api_key || secrets.STORE_API_KEY }} | |
| SEARCH_API_KEY: "${{ secrets.SEARCH_API_KEY }}" | |
| ASTRA_DB_APPLICATION_TOKEN: "${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}" | |
| ASTRA_DB_API_ENDPOINT: "${{ secrets.ASTRA_DB_API_ENDPOINT }}" | |
| ANTHROPIC_API_KEY: "${{ secrets.ANTHROPIC_API_KEY }}" | |
| TAVILY_API_KEY: "${{ secrets.TAVILY_API_KEY }}" | |
| UV_CACHE_DIR: /tmp/.uv-cache | |
| outputs: | |
| failed: ${{ steps.check-failure.outputs.failed }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.ref || github.ref }} | |
| - name: Setup Node.js Environment | |
| uses: actions/setup-node@v4 | |
| id: setup-node | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: "npm" | |
| cache-dependency-path: ./src/frontend/package-lock.json | |
| - name: Install Frontend Dependencies | |
| run: npm ci | |
| working-directory: ./src/frontend | |
| - name: Install Playwright Browser Dependencies | |
| id: install-playwright | |
| uses: ./.github/actions/install-playwright | |
| with: | |
| working-directory: ./src/frontend | |
| browsers: chromium | |
| - name: Setup Python Environment with UV | |
| uses: ./.github/actions/setup-uv | |
| - name: Install Python Dependencies | |
| run: uv sync --dev | |
| - name: Configure Environment Variables | |
| run: | | |
| touch .env | |
| echo "${{ secrets.ENV_VARS }}" > .env | |
| - name: Execute Playwright Tests | |
| uses: nick-fields/retry@v3 | |
| with: | |
| timeout_minutes: 12 | |
| max_attempts: 2 | |
| command: | | |
| cd src/frontend | |
| echo 'Running tests with pattern: ${{ needs.determine-test-suite.outputs.test_grep }}' | |
| npx playwright test ${{ inputs.tests_folder }} ${{ needs.determine-test-suite.outputs.test_grep }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --list | |
| # echo command before running | |
| echo "npx playwright test ${{ inputs.tests_folder }} ${{ needs.determine-test-suite.outputs.test_grep }} --trace on --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --workers 2" | |
| npx playwright test ${{ inputs.tests_folder }} ${{ needs.determine-test-suite.outputs.test_grep }} --trace on --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --workers 2 | |
| - name: Upload Test Results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: blob-report-${{ matrix.shardIndex }} | |
| path: src/frontend/blob-report | |
| retention-days: 1 | |
| - name: Cleanup UV Cache | |
| run: uv cache prune --ci | |
| merge-reports: | |
| # We need to repeat the condition at every step | |
| # https://github.com/actions/runner/issues/662 | |
| needs: setup-and-test | |
| runs-on: ubuntu-latest | |
| if: always() | |
| env: | |
| EXIT_CODE: ${{!contains(needs.setup-and-test.result, 'failure') && !contains(needs.setup-and-test.result, 'cancelled') && '0' || '1'}} | |
| steps: | |
| - name: "Should Merge Reports" | |
| # If the CI was successful, we don't need to merge the reports | |
| # so we can skip all the steps below | |
| id: should_merge_reports | |
| run: | | |
| if [ "$EXIT_CODE" == "0" ]; then | |
| echo "should_merge_reports=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "should_merge_reports=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Checkout code | |
| if: ${{ steps.should_merge_reports.outputs.should_merge_reports == 'true' }} | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| if: ${{ steps.should_merge_reports.outputs.should_merge_reports == 'true' }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Download blob reports from GitHub Actions Artifacts | |
| if: ${{ steps.should_merge_reports.outputs.should_merge_reports == 'true' }} | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: all-blob-reports | |
| pattern: blob-report-* | |
| merge-multiple: true | |
| - name: Merge into HTML Report | |
| if: ${{ steps.should_merge_reports.outputs.should_merge_reports == 'true' }} | |
| run: | | |
| npx playwright merge-reports --reporter html ./all-blob-reports | |
| - name: Upload HTML report | |
| if: ${{ steps.should_merge_reports.outputs.should_merge_reports == 'true' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: html-report--attempt-${{ github.run_attempt }} | |
| path: playwright-report | |
| retention-days: 14 |