Weekly data update #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Weekly data update | |
| on: | |
| schedule: | |
| - cron: '0 4 * * 1' # Every Monday at 04:00 UTC | |
| workflow_dispatch: # Also runnable manually from the GitHub Actions tab | |
| permissions: | |
| contents: write # commit updated data files | |
| issues: write # open quality gate issues | |
| pages: write # deploy to GitHub Pages | |
| id-token: write # required by deploy-pages | |
| concurrency: | |
| group: weekly-update | |
| cancel-in-progress: false | |
| jobs: | |
| update-data: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v5 | |
| - name: Install Python 3.13 | |
| run: uv python install 3.13 | |
| - name: Install dependencies | |
| run: uv sync | |
| # --- Municipalities --- | |
| - name: Scan Nordic municipalities (DNS) | |
| run: uv run preprocess DK FI NO SE IS FO GL | |
| - name: Postprocess (overrides, SMTP banners, scraping) | |
| run: uv run postprocess | |
| - name: Validate data quality | |
| id: validate | |
| continue-on-error: true | |
| run: uv run validate | |
| # --- Stock index companies --- | |
| # Runs regardless of municipality quality gate — company data is independent | |
| - name: Scan Nordic stock indices (OBX, OMXS30, OMXH25, OMXI15) | |
| run: uv run python3 scripts/classify_nordic_indices.py | |
| - name: Scan Danish stock index (OMXC20) | |
| run: uv run python3 scripts/classify_omxc20.py | |
| # --- Certificate Authority scan --- | |
| - name: Scan TLS certificates — municipalities (CA sovereignty) | |
| if: steps.validate.outcome == 'success' | |
| run: uv run scan-certs --skip-ct --timeout 20 | |
| - name: Scan TLS certificates — companies (CA sovereignty) | |
| run: uv run python3 scripts/scan_companies_ca.py --skip-ct --timeout 20 | |
| # --- Frontend --- | |
| - name: Build MX frontend data files | |
| if: steps.validate.outcome == 'success' | |
| run: uv run python3 scripts/build_frontend.py | |
| - name: Build CA frontend data files | |
| if: steps.validate.outcome == 'success' | |
| run: uv run python3 scripts/build_ca_frontend.py | |
| - name: Build companies CA frontend data files | |
| run: uv run python3 scripts/build_companies_ca_frontend.py | |
| # --- Commit & push --- | |
| # Stage company files unconditionally — they're independent of municipality quality | |
| - name: Stage company data | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add \ | |
| companies-ca-data.json \ | |
| companies-ca-summary.json \ | |
| companies-ca-detail.json \ | |
| data/obx_classified.json \ | |
| data/omxc20_classified.json \ | |
| data/omxs30_classified.json \ | |
| data/omxh25_classified.json \ | |
| data/omxi15_classified.json | |
| # Stage municipality files only when quality gate passed | |
| - name: Stage municipality data | |
| if: steps.validate.outcome == 'success' | |
| run: | | |
| git add \ | |
| data.json \ | |
| data-regions.json \ | |
| data-summary.json \ | |
| data-detail.json \ | |
| ca-data.json \ | |
| ca-data-summary.json \ | |
| ca-data-detail.json \ | |
| data/summary/ \ | |
| data/dns_cache/ \ | |
| validation_report.json \ | |
| validation_report.csv | |
| - name: Commit and push if anything changed | |
| run: | | |
| git diff --cached --quiet && echo "No data changes, skipping commit." && exit 0 | |
| git commit -m "chore: weekly data update $(date -u +%Y-%m-%d)" | |
| git pull --rebase | |
| git push | |
| - name: Open issue on quality gate failure | |
| if: steps.validate.outcome == 'failure' | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| existing=$(gh issue list --label quality-gate --state open --limit 1 --json number -q '.[0].number') | |
| if [ -n "$existing" ]; then | |
| gh issue comment "$existing" \ | |
| --body "Quality gate failed again on $(date -u +%Y-%m-%d). See [run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." | |
| else | |
| gh issue create \ | |
| --title "Quality gate failure: weekly data update $(date -u +%Y-%m-%d)" \ | |
| --label quality-gate \ | |
| --body "The weekly pipeline failed validation on $(date -u +%Y-%m-%d). See [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details." | |
| fi | |
| - name: Fail workflow on quality gate failure | |
| if: steps.validate.outcome == 'failure' | |
| run: exit 1 | |
| deploy: | |
| needs: update-data | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| group: pages | |
| cancel-in-progress: true | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| - uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: . | |
| - id: deployment | |
| uses: actions/deploy-pages@v4 |