Update Daily Data (EOD) #424
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Daily Data (EOD) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| force: | |
| description: 'Run even if market is open (for debugging)' | |
| required: false | |
| default: 'false' | |
| push: | |
| branches: [ main ] | |
| paths: | |
| - 'scripts/**' | |
| - 'requirements.txt' | |
| - '.github/workflows/update-daily.yml' | |
| schedule: | |
| - cron: '15 * * * *' | |
| permissions: | |
| contents: write | |
| concurrency: | |
| group: data-branch-publish | |
| cancel-in-progress: false | |
| jobs: | |
| update: | |
| runs-on: ubuntu-latest | |
| env: | |
| FORCE_RUN: ${{ github.event_name == 'push' && '1' || '0' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'npm' | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: 'requirements.txt' | |
| - name: Install Python deps | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: "Gate: decide whether to run (ET)" | |
| id: gate | |
| shell: bash | |
| run: | | |
| python - <<'PY' >> "$GITHUB_OUTPUT" | |
| import datetime | |
| import os | |
| from zoneinfo import ZoneInfo | |
| et = ZoneInfo("America/New_York") | |
| now = datetime.datetime.now(tz=et) | |
| force = ("${{ github.event.inputs.force }}" or "false").lower() == "true" | |
| if os.environ.get("FORCE_RUN") == "1": | |
| print("run=1") | |
| print("reason=forced_by_push") | |
| raise SystemExit(0) | |
| # Daily is intended once per day after market close. | |
| run = True | |
| reason = "ok" | |
| if not force: | |
| if now.weekday() >= 5: | |
| run = False | |
| reason = "weekend" | |
| else: | |
| # allow only after 16:05 ET | |
| mins = now.hour * 60 + now.minute | |
| if mins < (16*60 + 5): | |
| run = False | |
| reason = "before_close_buffer" | |
| print(f"run={1 if run else 0}") | |
| print(f"reason={reason}") | |
| PY | |
| - name: Check if already updated (ET day) | |
| id: already | |
| run: | | |
| set -euo pipefail | |
| TODAY_ET=$(TZ=America/New_York date +%Y-%m-%d) | |
| echo "TODAY_ET=$TODAY_ET" | |
| # Try to read existing daily_latest.json from data branch | |
| rm -rf _data_branch || true | |
| git clone --depth 1 --branch data https://github.com/${{ github.repository }}.git _data_branch || true | |
| if [ -f _data_branch/public/data/daily_latest.json ]; then | |
| EXISTING_DAY=$(node -e "const fs=require('fs'); const j=JSON.parse(fs.readFileSync('_data_branch/public/data/daily_latest.json','utf8')); const d=j?.daily?.meta?.lastTradingDay || j?.daily?.meta?.asOf || ''; console.log(String(d).slice(0,10));" || true) | |
| echo "EXISTING_DAY=$EXISTING_DAY" | |
| if [ "$EXISTING_DAY" = "$TODAY_ET" ]; then | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| echo "Already updated for $TODAY_ET → skipping." | |
| exit 0 | |
| fi | |
| fi | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| - name: Stop if gated | |
| if: steps.gate.outputs.run != '1' | |
| run: echo "Gated (${{ steps.gate.outputs.reason }}). Skipping update." && exit 0 | |
| - name: Build daily_latest.json | |
| run: python scripts/update_daily.py | |
| - name: Publish daily to data branch | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| rm -rf _data_branch | |
| git clone --depth 1 --branch data "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" _data_branch | |
| cd _data_branch | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| mkdir -p public/data | |
| cp -f ../public/data/daily_latest.json public/data/daily_latest.json | |
| git add public/data/daily_latest.json | |
| if git diff --cached --quiet; then | |
| echo "No changes to commit." | |
| exit 0 | |
| fi | |
| git commit -m "data(daily): refresh daily_latest.json [skip ci]" | |
| for attempt in 1 2 3; do | |
| echo "Push attempt $attempt..." | |
| git fetch origin data --depth=50 || true | |
| git rebase origin/data || true | |
| if git push origin HEAD:data; then | |
| echo "Push succeeded." | |
| exit 0 | |
| fi | |
| echo "Push rejected; retrying..." | |
| sleep $((attempt*2)) | |
| done | |
| echo "Push failed after retries." | |
| exit 1 |