Skip to content

feat(1681): patch-liveness report plugin for dead-mock detection #3457

feat(1681): patch-liveness report plugin for dead-mock detection

feat(1681): patch-liveness report plugin for dead-mock detection #3457

Workflow file for this run

name: CI
on:
push:
branches: [main, "epic/**"]
pull_request:
branches: [main, "epic/**"]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
pull-requests: read
jobs:
changes:
runs-on: ubuntu-latest
outputs:
benchmark: ${{ steps.filter.outputs.benchmark }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- id: filter
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d
with:
filters: |
benchmark:
- "src/file_organizer/cli/benchmark.py"
- "tests/ci/test_benchmark_contracts.py"
- "tests/cli/test_benchmark*.py"
- "tests/e2e/**"
- "tests/fixtures/benchmark*"
- "tests/fixtures/benchmark*/**"
- "docs/admin/performance-tuning.md"
- "docs/cli-reference.md"
- ".github/workflows/ci.yml"
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
# fetch-depth: 0 is required so origin/main is resolvable —
# pre-commit runs the tests/ci guardrails, which diff against it.
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.12"
cache: pip
- run: pip install pre-commit
- run: pip install -e ".[dev]"
- run: pre-commit run --all-files
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
unused-deps:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.12"
cache: pip
- run: pip install "deptry>=0.16.0"
- run: pip install -e .
- run: deptry src/
type-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.12"
cache: pip
- run: pip install -e ".[dev]"
- name: Run mypy on all of src/file_organizer/
run: mypy src/file_organizer/
link-integrity:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Check documentation links
run: |
echo "Checking for broken documentation links..."
# Extract markdown links from README and docs
links=$(grep -r -h "\[.*\](.*)" README.md docs/*.md 2>/dev/null | sed -E 's/.*\(([^)]+)\).*/\1/' | grep -E "^\/" | sort -u)
failed=0
for link in $links; do
# Skip external links and anchors
if [[ "$link" == http* ]]; then
continue
fi
# Remove anchor parts
filepath=${link%#*}
# Check if file exists
if [[ ! -f "$filepath" ]]; then
echo "❌ Broken link: $link"
failed=1
fi
done
if [ $failed -eq 1 ]; then
echo "Some documentation links are broken!"
exit 1
fi
echo "✅ All documentation links are valid"
test:
name: "Test PR suite (py${{ matrix.python-version }})"
needs: changes
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
matrix:
python-version: ["3.11", "3.12", "3.13", "3.14"]
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
# fetch-depth: 0 is required for the diff-cover step on PRs —
# shallow clones lack the merge-base needed to compute the changed-line diff.
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: ${{ matrix.python-version }}
cache: pip
- name: Install dependencies
run: |
# Install core package with dev dependencies + search (rank-bm25, scikit-learn)
# search is required for TestBM25Index, TestVectorIndexRecall, TestHybridRetriever,
# and related CLI/API/copilot tests — not in dev because it is an optional runtime extra.
# parsers/archive/scientific/cad supply the optional reader deps (PyMuPDF, python-docx,
# openpyxl, python-pptx, ebooklib, py7zr, rarfile, h5py, netCDF4, scipy, ezdxf) so the
# SafeDir reader fileobj branches are exercised here rather than skipped — required for
# the diff-coverage gate on tests/utils/test_readers_safedir.py (WP-2.1).
pip install -e ".[dev,search,parsers,archive,scientific,cad]"
# Explicitly install pytest-asyncio and faker to ensure they're available
# This fixes installation order issues on Python 3.11 where pytest might load before pytest-asyncio
pip install "pytest-asyncio>=0.23.0" faker --no-cache-dir
- name: Verify test dependencies
run: |
# Fast plugin registration check — replaces the former pytest --collect-only
# scan which took 5-6 minutes on CI to import all test files.
python -c "
import importlib.metadata, pytest_asyncio, faker
plugins = {ep.name for ep in importlib.metadata.entry_points(group='pytest11')}
assert 'asyncio' in plugins, f'pytest-asyncio not registered as pytest plugin; found: {plugins}'
print(f'pytest-asyncio {pytest_asyncio.__version__} registered as plugin')
print(f'faker {importlib.metadata.version(\"faker\")} importable')
"
- name: Run tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: pytest tests/ --strict-markers -m "ci and not benchmark" --cov=file_organizer --cov-report=xml --timeout=30 -n=auto --override-ini="addopts="
- name: Diff coverage gate
if: ${{ github.event.pull_request.changed_files <= 75 }}
# Checks that lines added/changed in this PR are ≥80% covered.
# Uses the ci-suite coverage.xml generated above — fast, no full-suite re-run.
# Ratchet: bump --fail-under as coverage improves. Target: 90% per issue #855.
# Broad refactors and type-cleanup sweeps exceed the signal-to-noise ratio
# where diff coverage is useful, so we skip the gate for very large PRs.
run: |
pip install diff-cover --quiet
diff-cover coverage.xml --compare-branch=origin/main --fail-under=80
test-full:
name: "Test (shard ${{ matrix.shard }}, py${{ matrix.python-version }})"
needs: changes
if: github.event_name == 'push'
runs-on: ubuntu-latest
# Domain-based shard split. After fixing realtime queue-task cleanup,
# all shards can use xdist again instead of the emergency micro-shard matrix.
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
python-version: ["3.11", "3.12", "3.13", "3.14"]
shard: [1, 2, 3, 4, 5, 6]
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
# fetch-depth: 0 so tests/ci guardrails can resolve origin/main for their
# diff (shallow checkout leaves it unreachable, e.g. on epic/** branches).
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: ${{ matrix.python-version }}
cache: pip
- name: Install dependencies
run: |
pip install -e ".[dev,search]"
pip install "pytest-asyncio>=0.23.0" faker --no-cache-dir
- name: Run test shard
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Shard → directory mapping lives in scripts/ci_shard_paths.sh so both
# ci.yml and ci-full.yml stay in sync from a single source of truth.
PATHS=$(bash scripts/ci_shard_paths.sh "${{ matrix.shard }}")
pytest $PATHS --strict-markers -m "not benchmark and not e2e" \
--cov=file_organizer --cov-report= \
--timeout=30 -n=auto --override-ini="addopts="
# Rename .coverage so artifacts from all shards have distinct names
mv .coverage .coverage.shard-${{ matrix.shard }}
- name: Upload coverage data
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: coverage-${{ matrix.python-version }}-${{ matrix.shard }}
path: .coverage.shard-${{ matrix.shard }}
include-hidden-files: true
retention-days: 1
playwright:
# Matrix execution policy: all three browsers run on every PR and push.
# Rationale: catching firefox/webkit-only regressions at PR review time is
# the entire point of cross-browser testing — chromium-only on PRs would
# let those regressions land and be discovered later by nightly runs (or
# users). The three browsers run in parallel, so wall-clock cost is ~1x
# the chromium-only run, only the runner-minute cost is ~3x. fail-fast is
# disabled so a firefox failure does not cancel the in-progress webkit run
# (review value of seeing all three results > saving a few runner minutes).
name: "Playwright E2E (${{ matrix.browser }})"
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
browser: [chromium, firefox, webkit]
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.12"
cache: pip
- name: Cache Playwright virtualenv
id: playwright-venv-cache
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: .venv-playwright
# Shared across all three browser legs: they run the same Python env.
key: playwright-venv-${{ runner.os }}-py3.12-${{ hashFiles('**/pyproject.toml') }}
restore-keys: playwright-venv-${{ runner.os }}-py3.12-
- name: Install dependencies
if: steps.playwright-venv-cache.outputs.cache-hit != 'true'
run: |
python -m venv .venv-playwright
. .venv-playwright/bin/activate
python -m pip install --upgrade pip
# [dev] provides pytest-playwright; [search] keeps parity with other
# test jobs so imports don't break at collection time.
pip install -e ".[dev,search]"
# Retry plugin for per-test flake tolerance. Installed here (not in
# pyproject.toml) because only this job needs it.
pip install "pytest-rerunfailures>=14.0" --no-cache-dir
- name: Cache Playwright browsers
id: playwright-cache
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: ~/.cache/ms-playwright
# Browser is part of the cache key so chromium/firefox/webkit caches
# do not clobber each other across matrix legs.
key: playwright-${{ runner.os }}-${{ matrix.browser }}-${{ hashFiles('**/pyproject.toml') }}
restore-keys: playwright-${{ runner.os }}-${{ matrix.browser }}-
- name: Install Playwright browser
# --with-deps pulls the system libs the browser needs on a fresh runner.
run: |
. .venv-playwright/bin/activate
python -m playwright install --with-deps ${{ matrix.browser }}
- name: Run Playwright suite
# Overriding addopts is required because pyproject.toml's default
# addopts filter out e2e/playwright-marked tests in normal runs.
# --reruns 2 tolerates one-off browser flakes; real failures still
# surface on the third attempt. Traces/screenshots/videos are
# retained only on failure to keep artifact size manageable.
run: |
. .venv-playwright/bin/activate
pytest tests/playwright/ \
--browser ${{ matrix.browser }} \
--tracing retain-on-failure \
--screenshot only-on-failure \
--video retain-on-failure \
--output=playwright-artifacts \
--reruns 2 --reruns-delay 2 \
--timeout=60 \
--strict-markers \
--override-ini="addopts="
- name: Upload Playwright artifacts on failure
if: failure()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
# Browser name in artifact name prevents matrix legs from clobbering
# each other when uploading to the same workflow run.
name: playwright-artifacts-${{ matrix.browser }}
path: playwright-artifacts/
retention-days: 7
if-no-files-found: ignore
coverage-gate:
name: "Coverage gate (py3.11)"
needs: test-full
if: github.event_name == 'push'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.11"
cache: pip
- run: pip install -e ".[dev]"
- name: Download coverage data (py3.11 shards)
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: coverage-3.11-*
merge-multiple: true
- name: Combine and enforce gate
run: |
coverage combine .coverage.shard-*
coverage report --fail-under=93
coverage xml
- name: Check docstring coverage
run: interrogate -v src/ --fail-under 95
- name: Upload to Codecov
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
file: coverage.xml
fail_ci_if_error: false
test-benchmark:
name: Benchmark suite
needs: changes
if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.benchmark == 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read # required by actions/checkout
pull-requests: write # post advisory comparison comment on PRs
actions: read # download benchmark-baseline artifact from prior main runs
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.11"
cache: pip
- name: Install dependencies
run: pip install -e ".[dev]"
- name: Download benchmark baseline (PR only)
# Finds the latest successful main CI run that stored a benchmark-baseline
# artifact and downloads it for comparison. Skipped silently if none exists.
if: github.event_name == 'pull_request'
continue-on-error: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
RUN_ID=$(gh api \
"repos/${{ github.repository }}/actions/workflows/ci.yml/runs?branch=main&status=success&per_page=10" \
--jq '[.workflow_runs[] | select(.conclusion=="success")][0].id' 2>/dev/null || echo "")
if [ -n "$RUN_ID" ] && [ "$RUN_ID" != "null" ]; then
gh run download "$RUN_ID" --name benchmark-baseline --dir benchmark-baseline/ 2>/dev/null \
&& echo "Baseline downloaded from main run $RUN_ID" \
|| echo "No benchmark-baseline artifact in run $RUN_ID — first-time run"
else
echo "No successful main run found — skipping baseline comparison"
fi
- name: Run benchmark suite
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
OUT="benchmark-results-pr.json"
else
OUT="benchmark-results.json"
fi
pytest tests/ -m benchmark --benchmark-only --strict-markers --timeout=30 \
--benchmark-json="$OUT" --override-ini="addopts="
- name: Compare against baseline (PR only, advisory)
# Posts a markdown comment on the PR showing mean-time changes per benchmark.
# Advisory only — does not block merge. Threshold: 🔴 >10%, 🟡 >5%, 🟢 ≤5%.
if: github.event_name == 'pull_request'
continue-on-error: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
BASELINE="benchmark-baseline/benchmark-results.json"
CURRENT="benchmark-results-pr.json"
if [ ! -f "$BASELINE" ] || [ ! -f "$CURRENT" ]; then
echo "Skipping comparison — baseline or current results not available"
exit 0
fi
cat > /tmp/compare_benchmarks.py << 'PYEOF'
import json, sys, importlib.metadata
try:
with open(sys.argv[1]) as f:
baseline = {b["name"]: b["stats"]["mean"] for b in json.load(f)["benchmarks"]}
with open(sys.argv[2]) as f:
current = {b["name"]: b["stats"]["mean"] for b in json.load(f)["benchmarks"]}
except (KeyError, json.JSONDecodeError) as e:
print(f"Skipping comparison — could not parse benchmark results: {e}")
sys.exit(0)
rows, regressions = [], []
for name in sorted(set(baseline) & set(current)):
base, cur = baseline[name], current[name]
pct = (cur - base) / base * 100
icon = "🔴" if pct > 10 else ("🟡" if pct > 5 else "🟢")
rows.append(f"| `{name}` | {base*1000:.2f} ms | {cur*1000:.2f} ms | {pct:+.1f}% {icon} |")
if pct > 10:
regressions.append(name)
summary = "✅ No significant regressions" if not regressions \
else f"⚠️ {len(regressions)} benchmark(s) regressed >10%: {', '.join(f'`{n}`' for n in regressions)}"
body = f"""### Benchmark Comparison (advisory)
{summary}
| Benchmark | Baseline | PR | Change |
|-----------|----------|-----|--------|
{chr(10).join(rows) if rows else "| (no benchmarks in common) | — | — | — |"}
> Advisory only — does not block merge.
<!-- benchmark-comment -->
"""
with open("/tmp/benchmark-comment.md", "w") as f:
f.write("\n".join(line.lstrip() for line in body.splitlines()))
PYEOF
python3 /tmp/compare_benchmarks.py "$BASELINE" "$CURRENT"
# Use marker-based update so we always edit the benchmark comment
# specifically, not whichever automation comment happened to be last.
PR_NUM="${{ github.event.pull_request.number }}"
COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${PR_NUM}/comments" \
--jq '[.[] | select(.body | contains("<!-- benchmark-comment -->"))][0].id // empty')
if [ -n "$COMMENT_ID" ]; then
gh api --method PATCH "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \
--field body="$(cat /tmp/benchmark-comment.md)"
else
gh pr comment "$PR_NUM" --body-file /tmp/benchmark-comment.md
fi
- name: Upload benchmark results (main only)
if: github.event_name == 'push'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: benchmark-baseline
path: benchmark-results.json
retention-days: 30
test-integration:
name: Integration coverage gate
needs: changes
if: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.11"
cache: pip
- name: Install dependencies
run: pip install -e ".[dev,search]"
- name: "Integration coverage gate (floor: 76.5% combined line+branch)"
# Ratchet floor — bump threshold in the PR that closes each gap.
# 2026-03-17: 35.43% line (173 integration tests, clean CI env with [dev,search] extras).
# 2026-03-20: 30% branch (ratchet after enabling branch = true; issue #912). Target per issue #856: 90%.
# 2026-03-21: 45% branch (ratchet after #945/#947 branch-coverage work; actual 49%).
# 2026-03-22: 50% combined (ratchet after PR1 cli+models tests; actual 50.9%; 2783 integration tests).
# 2026-03-22: 55% combined (ratchet after PR2 api+web tests; actual 55%; 3311 integration tests).
# 2026-03-22: 56% combined (ratchet after PR3 web+plugins tests; actual 56.64%; 3507 integration tests).
# 2026-03-22: 60% combined (ratchet after PR4 services+review_regressions+benchmark+cli tests; actual 60.06%; 4112 integration tests).
# 2026-04-05: 71.9% combined (ratchet after batch3/batch4 integration coverage expansion on main).
# 2026-04-06: floor held at 71.9% — new integration tests added (auth+service-facade+websocket); local measurement 71.76% due to missing [search] extras in local venv vs CI env with [dev,search]
# 2026-04-06: 72.0% combined (ratchet after dedupe-hash/removal + repo integration tests; actual 72%)
# 2026-06-25: 75.7% combined (ratchet after ebook, document, audio, video and model coverage updates; actual 75.77%)
# 2026-07-19: 76.0% combined (ratchet after genuine integration tests for the JD migrator, dedupe CLI, and deploy scaling/monitoring/compose modules; issue #1582)
# 2026-07-19: 76.5% combined (ratchet after genuine integration tests for the updater check→install→rollback cycle, desktop launcher daemon boot, and TUI Pilot view transitions; issue #1582)
# 2026-07-23: count the conformance suite (#1599 parity slice). It drives the canonical
# service and every adapter end to end, so it is integration-level evidence for the new
# core/tui/methodology modules; it stays separately gated by the Conformance workflow. The
# floor is regenerated from this pool and ratchets in the same PR (#1628 coverage follow-up).
run: pytest tests/ -m "integration or conformance" --strict-markers --cov=file_organizer --cov-branch --cov-fail-under=76.5 --cov-report=term-missing --cov-report=xml --cov-report=json:.coverage-integration.json --timeout=60 --override-ini="addopts="
- name: Check per-file integration coverage floors
run: python scripts/coverage/check-integration-floors.py
test-unit-floors:
name: Unit coverage floor gate
needs: changes
if: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.11"
cache: pip
- name: Install dependencies
run: pip install -e ".[dev,search]"
- name: Unit coverage run
run: pytest tests/ -m "unit" --strict-markers --cov=file_organizer --cov-branch --cov-fail-under=0 --cov-report=json:.coverage-unit.json --timeout=60 --override-ini="addopts="
- name: Upload unit coverage JSON
# The artifact is the source of truth for regenerating
# [tool.coverage.floors.unit] (generate_module_coverage_floor.py):
# floors must be reconciled against CI's own environment, not a
# contributor's local run.
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: coverage-unit-json
path: .coverage-unit.json
retention-days: 30
# Dot-prefixed paths are "hidden" and excluded by default since
# upload-artifact v4.4; without this the artifact silently uploads nothing.
include-hidden-files: true
- name: Check per-file unit coverage floors
run: python scripts/coverage/check_module_coverage_floor.py