deplyment guide maj #256
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main, master] | |
| pull_request: | |
| branches: [main, master] | |
| workflow_dispatch: | |
| inputs: | |
| run_golden: | |
| description: "Run golden tests (if GeneWeb present)" | |
| type: boolean | |
| default: false | |
| # Make failures visible quickly and avoid hangs | |
| permissions: | |
| contents: read | |
| jobs: | |
| filters: | |
| name: Detect golden-relevant changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| golden: ${{ steps.filter.outputs.golden }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Paths filter | |
| id: filter | |
| uses: dorny/paths-filter@v3 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| filters: | | |
| golden: | |
| - 'GeneWeb/**' | |
| - 'scripts/golden/**' | |
| - 'tests/golden/**' | |
| - 'python_app/routes/**' | |
| - 'python_app/migrated/**' | |
| quality: | |
| name: Quality Gates (format + lint + types + security) | |
| runs-on: macos-latest | |
| timeout-minutes: 30 | |
| env: | |
| LANG: en_US.UTF-8 | |
| LC_ALL: en_US.UTF-8 | |
| TZ: UTC | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.11"] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: pip | |
| - name: Install project dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| - name: Install quality toolchain | |
| run: | | |
| pip install \ | |
| black \ | |
| pylint \ | |
| mypy \ | |
| ruff \ | |
| bandit[toml] \ | |
| pip-audit \ | |
| pytest pytest-cov | |
| - name: Enable Pylint problem matcher | |
| run: echo "::add-matcher::.github/matchers/pylint.json" | |
| - name: Cache mypy | |
| uses: actions/cache@v4 | |
| with: | |
| path: .mypy_cache | |
| key: mypy-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/*.py','mypy.ini','.pylintrc','pyproject.toml','setup.cfg') }} | |
| - name: Lint - Ruff (fast static checks) | |
| run: | | |
| set -euo pipefail | |
| ruff check python_app tests/python | |
| - name: Code style - Black | |
| run: | | |
| set -euo pipefail | |
| black --check python_app tests/python | |
| - name: Lint - Pylint (strict) | |
| run: | | |
| set -euo pipefail | |
| pylint -j 2 python_app tests/python | |
| - name: Type check - Mypy | |
| run: | | |
| set -euo pipefail | |
| # Only check production code (python_app/) - tests have duplicate module names | |
| # and more relaxed typing requirements | |
| mypy python_app | |
| - name: Security - Dependency audit (pip-audit) | |
| run: | | |
| set -euo pipefail | |
| pip-audit | |
| - name: Security - Bandit (SAST) | |
| run: | | |
| set -euo pipefail | |
| bandit -q -r python_app || (echo "Bandit found issues"; exit 1) | |
| tests: | |
| name: Tests + Coverage (macOS) | |
| runs-on: macos-latest | |
| timeout-minutes: 45 | |
| needs: [quality, filters] | |
| env: | |
| LANG: en_US.UTF-8 | |
| LC_ALL: en_US.UTF-8 | |
| TZ: UTC | |
| RUN_GOLDEN: ${{ (inputs.run_golden == true) || (needs.filters.outputs.golden == 'true') || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'golden')) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: pip | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| pip install pytest pytest-cov | |
| - name: Check if we need to build GeneWeb from source | |
| id: check_source | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [ -d "GeneWeb" ]; then | |
| echo "has_binaries=true" >> $GITHUB_OUTPUT | |
| echo "needs_build=false" >> $GITHUB_OUTPUT | |
| echo "✅ GeneWeb binaries found" | |
| elif [ -d "source_geneweb" ]; then | |
| echo "has_binaries=false" >> $GITHUB_OUTPUT | |
| echo "needs_build=true" >> $GITHUB_OUTPUT | |
| echo "🔨 Will build GeneWeb from source" | |
| else | |
| echo "has_binaries=false" >> $GITHUB_OUTPUT | |
| echo "needs_build=false" >> $GITHUB_OUTPUT | |
| echo "⚠️ No GeneWeb found - skipping OCaml tests" | |
| fi | |
| - name: Setup OCaml | |
| if: steps.check_source.outputs.needs_build == 'true' | |
| uses: ocaml/setup-ocaml@v3.4.0 | |
| with: | |
| ocaml-compiler: 4.14.2 | |
| - name: Cache OPAM | |
| if: steps.check_source.outputs.needs_build == 'true' | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.opam | |
| key: opam-${{ runner.os }}-${{ hashFiles('source_geneweb/**') }} | |
| - name: Install GeneWeb dependencies | |
| if: steps.check_source.outputs.needs_build == 'true' | |
| working-directory: source_geneweb | |
| env: | |
| OPAMYES: true | |
| run: | | |
| set -euo pipefail | |
| opam install . --deps-only --with-test | |
| - name: Configure GeneWeb | |
| if: steps.check_source.outputs.needs_build == 'true' | |
| working-directory: source_geneweb | |
| run: | | |
| set -euo pipefail | |
| opam exec -- ocaml ./configure.ml --sosa-zarith | |
| - name: Build GeneWeb | |
| if: steps.check_source.outputs.needs_build == 'true' | |
| working-directory: source_geneweb | |
| run: | | |
| set -euo pipefail | |
| opam exec -- make distrib | |
| echo "✅ Build complete" | |
| - name: Setup GeneWeb runtime from distribution | |
| if: steps.check_source.outputs.needs_build == 'true' | |
| run: | | |
| set -euo pipefail | |
| if [ -d "source_geneweb/distribution" ]; then | |
| mv source_geneweb/distribution GeneWeb | |
| echo "✅ GeneWeb runtime ready from distribution" | |
| else | |
| echo "❌ Distribution folder not found" | |
| exit 1 | |
| fi | |
| if [ -d "GeneWeb/bases/test.gwb" ]; then | |
| echo "✅ Test database found" | |
| else | |
| echo "⚠️ Test database not found - some tests may fail" | |
| fi | |
| - name: Make executables | |
| if: steps.check_source.outputs.has_binaries == 'true' || steps.check_source.outputs.needs_build == 'true' | |
| run: | | |
| set -euo pipefail | |
| chmod +x GeneWeb/gw/* || true | |
| chmod +x scripts/golden/*.sh || true | |
| - name: Smoke test gwd | |
| if: steps.check_source.outputs.has_binaries == 'true' || steps.check_source.outputs.needs_build == 'true' | |
| working-directory: GeneWeb | |
| run: | | |
| set -euo pipefail | |
| # Ensure parent etc directory exists so gwd can create per-base dirs | |
| mkdir -p ./bases/etc | |
| ./gw/gwd -hd ./gw -bd ./bases -p 23179 -lang en > gwd.out 2>&1 & | |
| GWD_PID=$! | |
| # Give gwd extra time to initialize | |
| sleep 6 | |
| # Determine actual base name present (test or base) | |
| BASE_NAME="test" | |
| if [ ! -d ./bases/test.gwb ] && [ -d ./bases/base.gwb ]; then | |
| BASE_NAME="base" | |
| fi | |
| set +e | |
| CODE=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:23179/${BASE_NAME}") | |
| set -e | |
| if [ "$CODE" = "000" ] || [ "$CODE" -ge 500 ]; then | |
| echo "Smoke check failed (HTTP $CODE). Tail of gwd.out:" | |
| tail -n 100 gwd.out || true | |
| kill "$GWD_PID" 2>/dev/null || true | |
| exit 1 | |
| fi | |
| URL="http://localhost:23179/${BASE_NAME}" | |
| curl -sf "$URL" | grep -qi "geneweb" || { echo "Marker not found on home"; tail -n 100 gwd.out; kill "$GWD_PID" 2>/dev/null || true; exit 1; } | |
| PERSON_URL="${URL}?p=Charles&n=Windsor" | |
| if ! curl -sf "$PERSON_URL" | grep -qiE "name|person|id"; then | |
| echo "Person page markers missing on $PERSON_URL" | |
| tail -n 100 gwd.out || true | |
| kill "$GWD_PID" 2>/dev/null || true | |
| exit 1 | |
| fi | |
| FR_URL="${URL}?lang=fr" | |
| if ! curl -sf "$FR_URL" | grep -qiE "Accueil|Personne|Famille"; then | |
| echo "FR localization check failed on $FR_URL" | |
| tail -n 100 gwd.out || true | |
| kill "$GWD_PID" 2>/dev/null || true | |
| exit 1 | |
| fi | |
| echo "gwd smoke check passed (HTTP $CODE)" | |
| kill "$GWD_PID" 2>/dev/null || true | |
| - name: Export GEDCOM test | |
| if: steps.check_source.outputs.has_binaries == 'true' || steps.check_source.outputs.needs_build == 'true' | |
| working-directory: GeneWeb | |
| run: | | |
| set -euo pipefail | |
| if [ -d bases/test.gwb ]; then | |
| BASE_PATH="bases/test.gwb" | |
| elif [ -d bases/base.gwb ]; then | |
| BASE_PATH="bases/base.gwb" | |
| else | |
| echo "Export check failed: no test.gwb or base.gwb found" | |
| exit 1 | |
| fi | |
| ./gw/gwb2ged "$BASE_PATH" -o /tmp/export_test.ged | |
| test -s /tmp/export_test.ged | |
| rm -f /tmp/export_test.ged | |
| echo "Export check passed" | |
| # ---------- Python tests + coverage gate ---------- | |
| - name: Python infrastructure test | |
| run: | | |
| set -euo pipefail | |
| pytest tests/python/unit/test_setup.py -v | |
| - name: Python unit tests (coverage reported, not gating) | |
| run: | | |
| set -euo pipefail | |
| pytest -v --tb=short \ | |
| --cov=python_app --cov-report=xml --cov-report=term \ | |
| tests/python/unit | |
| - name: Python integration tests | |
| run: | | |
| set -euo pipefail | |
| pytest -v --tb=short tests/python/integration | |
| - name: Python functional tests | |
| run: | | |
| set -euo pipefail | |
| pytest -v --tb=short tests/python/functional | |
| - name: Python proxy server smoke test (non-blocking) | |
| continue-on-error: true | |
| run: | | |
| set -euo pipefail | |
| python -c "from python_app.config import Config; from python_app.ocaml_bridge import OCamlBridge; from python_app.app import app; print('✅ All imports successful')" || { echo "❌ Import failed"; exit 1; } | |
| python -c "from python_app.config import Config; Config.validate(); print('✅ Config validation passed')" || echo "⚠️ Config validation failed (may be OK if GeneWeb not available)" | |
| if [ -f "GeneWeb/gw/gwd" ] && [ -d "GeneWeb/bases/test.gwb" ]; then | |
| echo "Testing proxy server with OCaml backend..." | |
| cd GeneWeb | |
| ./gw/gwd -hd ./gw -bd ./bases -p 23180 -lang en > /tmp/gwd_proxy_test.log 2>&1 & | |
| GWD_PID=$! | |
| cd .. | |
| sleep 3 | |
| BACKEND=ocaml FLASK_PORT=23181 GENEWEB_DIR=./GeneWeb python -m python_app.app > /tmp/flask_proxy_test.log 2>&1 & | |
| FLASK_PID=$! | |
| sleep 2 | |
| curl -s http://localhost:23181/health | tee /tmp/proxy_health.json | |
| kill $FLASK_PID 2>/dev/null || true | |
| BACKEND=python FLASK_PORT=23181 GENEWEB_DIR=./GeneWeb python -m python_app.app > /tmp/flask_proxy_test.log 2>&1 & | |
| FLASK_PID=$! | |
| sleep 2 | |
| curl -s http://localhost:23181/health | tee /tmp/proxy_health_python.json | |
| kill $FLASK_PID 2>/dev/null || true | |
| kill $GWD_PID 2>/dev/null || true | |
| pkill -f "gwd.*-p 23180" 2>/dev/null || true | |
| pkill -f "python.*python_app" 2>/dev/null || true | |
| else | |
| echo "⚠️ Skipping proxy server runtime tests (GeneWeb binaries or test database not available)" | |
| fi | |
| echo "✅ Python proxy server smoke test complete" | |
| - name: Golden validate (optional) | |
| if: (steps.check_source.outputs.has_binaries == 'true' || steps.check_source.outputs.needs_build == 'true') && env.RUN_GOLDEN == 'true' | |
| run: | | |
| set -euo pipefail | |
| chmod +x ./scripts/golden/run_golden.sh || true | |
| bash ./scripts/golden/run_golden.sh validate | |
| - name: GEDCOM import roundtrip (optional) | |
| if: (steps.check_source.outputs.has_binaries == 'true' || steps.check_source.outputs.needs_build == 'true') && env.RUN_GOLDEN == 'true' | |
| continue-on-error: true | |
| run: | | |
| set -euo pipefail | |
| chmod +x ./scripts/golden/test_gedcom_import.sh || true | |
| bash ./scripts/golden/test_gedcom_import.sh validate | |
| # ---------- Artifacts ---------- | |
| - name: Upload coverage | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: python-coverage | |
| path: | | |
| coverage.xml | |
| htmlcov/ | |
| if-no-files-found: ignore | |
| - name: Upload pytest logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pytest-logs | |
| path: tests/python/pytest.log | |
| if-no-files-found: ignore | |
| - name: Upload golden diff on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: golden-diff | |
| path: | | |
| tests/golden/reports/diff.txt | |
| tests/golden/reports/import_diff.txt | |
| if-no-files-found: ignore |