Data Sync #128
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Data Sync | |
| # ────────────────────────────────────────────────────────────── | |
| # DART 메인 데이터 수집 (2026-04-16 정리) | |
| # | |
| # 자동 cron (recent 모드): | |
| # 1. DART list.json 으로 최근 N일 정기공시 조회 | |
| # 2. 카테고리별 독립 누락 검사 (docs rcept_no + finance reprt_code) | |
| # 3. 누락된 종목·분기만 수집 — 88분기 차집합 X | |
| # 4. 이전 실행 pending.txt (API 한도 잘림) 우선 회수 | |
| # 5. 변경 파일만 HF push | |
| # | |
| # 수동 (workflow_dispatch): | |
| # - mode=recent (기본): list.json 기반 경량 수집 (~30분) | |
| # - mode=full: 88분기 차집합 수집 (~5시간, 이전 dataPipeline full 모드 대체) | |
| # | |
| # 파이프라인 상세: engines.data | |
| # ────────────────────────────────────────────────────────────── | |
| on: | |
| schedule: | |
| - cron: '0 18 * * *' # UTC 18:00 = KST 03:00 (야간 공시 캡처) | |
| - cron: '0 6 * * *' # UTC 06:00 = KST 15:00 (당일 오전 공시 빠른 반영) | |
| workflow_dispatch: | |
| inputs: | |
| lookback_days: | |
| description: '신규 공시 조회 기간 (일)' | |
| default: '14' | |
| type: string | |
| mode: | |
| description: 'recent (list.json 기반) / full (88분기 차집합, 무거움)' | |
| default: 'recent' | |
| type: choice | |
| options: [recent, full] | |
| category: | |
| description: '(full 모드용) finance/report/docs/all' | |
| default: 'all' | |
| type: choice | |
| options: [all, finance, report, docs] | |
| permissions: | |
| contents: write | |
| # DART 경로 (dart/*) 에 push 하는 워크플로우만 직렬화. | |
| # 다른 경로 (edgar/*, kindlist/*, search/*) 는 별도 group — cancelled 연쇄 방지. | |
| concurrency: | |
| group: hf-dart-push | |
| cancel-in-progress: false | |
| jobs: | |
| # ── Job 1: finance + report (list.json 기반, ~30분) ── | |
| sync-finance-report: | |
| if: ${{ github.event.inputs.mode != 'full' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Set up Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync | |
| - name: Smoke import (fail fast on src regression) | |
| run: uv run python -c "import dartlab; print('dartlab', getattr(dartlab, '__version__', 'dev'))" | |
| - name: Restore finance cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: data/dart/finance | |
| key: dartlab-data-finance-${{ github.run_id }} | |
| restore-keys: dartlab-data-finance- | |
| - name: Restore report cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: data/dart/report | |
| key: dartlab-data-report-${{ github.run_id }} | |
| restore-keys: dartlab-data-report- | |
| - name: Restore docs cache (비교용) | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: data/dart/docs | |
| key: dartlab-data-docs-${{ github.run_id }} | |
| restore-keys: dartlab-data-docs- | |
| - name: Restore collect state (pending.txt) — fr scope | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: data/dart/_collect_state/fr | |
| key: dartlab-collect-state-fr-${{ github.run_id }} | |
| restore-keys: dartlab-collect-state-fr- | |
| # HF seed: GHA cache 가 cold/stale 해도 부족 parquet 을 HF 에서 채움 (idempotent). | |
| # cold start death spiral 차단 — HF 가 source of truth, GHA cache 는 속도 부스터. | |
| - name: Seed missing parquet from HF | |
| run: uv run python -X utf8 .github/scripts/sync/seedFromHf.py --category finance,report,docs | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| - name: Sync recent (finance + report) | |
| # -u + PYTHONUNBUFFERED=1: stdout 라인 버퍼링 차단 — 90 분 timeout cancel 시 | |
| # 어디서 멈췄는지 실시간 가시화. 이전 cancel run 84 분 동안 stdout 0 줄 = | |
| # buffered 패턴 (정상 종료 시에만 flush). 진척 진단 불가 사고 차단. | |
| run: uv run python -X utf8 -u .github/scripts/sync/syncRecent.py | |
| env: | |
| DART_API_KEYS: ${{ secrets.DART_API_KEYS }} | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} # checkpoint subprocess (uploadData.py) 가 필요 | |
| SYNC_LOOKBACK_DAYS: ${{ github.event.inputs.lookback_days || '14' }} | |
| SYNC_CATEGORIES: finance,report | |
| SYNC_STATE_SCOPE: fr # 병렬 Job pending.txt 경쟁 회피 | |
| PYTHONUNBUFFERED: "1" | |
| # Phase A — parquet → .arrow IPC mirror 빌드. Phase D 의 read_ipc(memory_map=True) | |
| # 진입점이 사용. mtime 비교로 변경된 parquet 만 재변환 (idempotent). | |
| - name: Build IPC mirror (finance + report) | |
| run: uv run python -X utf8 .github/scripts/sync/buildIpcMirror.py | |
| env: | |
| SYNC_CATEGORIES: finance,report | |
| - name: Upload to HuggingFace (finance) | |
| if: hashFiles('dist/changed_finance.txt') != '' | |
| run: | | |
| if [ -s dist/changed_finance.txt ]; then | |
| cp dist/changed_finance.txt dist/changed.txt | |
| uv run python .github/scripts/sync/uploadData.py --target hf | |
| fi | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| SYNC_CATEGORY: finance | |
| - name: Upload to HuggingFace (report) | |
| if: hashFiles('dist/changed_report.txt') != '' | |
| run: | | |
| if [ -s dist/changed_report.txt ]; then | |
| cp dist/changed_report.txt dist/changed.txt | |
| uv run python .github/scripts/sync/uploadData.py --target hf | |
| fi | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| SYNC_CATEGORY: report | |
| # GitHub Releases 업로드 폐지 (2026-04-08): HF만 유지. | |
| # data-finance/data-report 태그는 정리됨. uploadData.py --target gh 호출 금지. | |
| - name: Save finance cache | |
| if: always() | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: data/dart/finance | |
| key: dartlab-data-finance-${{ github.run_id }} | |
| - name: Save report cache | |
| if: always() | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: data/dart/report | |
| key: dartlab-data-report-${{ github.run_id }} | |
| - name: Save collect state — fr scope | |
| if: always() | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: data/dart/_collect_state/fr | |
| key: dartlab-collect-state-fr-${{ github.run_id }} | |
| # ── Job 2: docs (느림, 별도 타임아웃) ── | |
| sync-docs: | |
| if: ${{ github.event.inputs.mode != 'full' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Set up Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync | |
| - name: Smoke import (fail fast on src regression) | |
| run: uv run python -c "import dartlab; print('dartlab', getattr(dartlab, '__version__', 'dev'))" | |
| - name: Restore docs cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: data/dart/docs | |
| key: dartlab-data-docs-${{ github.run_id }} | |
| restore-keys: dartlab-data-docs- | |
| - name: Restore collect state (pending.txt) — docs scope | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: data/dart/_collect_state/docs | |
| key: dartlab-collect-state-docs-${{ github.run_id }} | |
| restore-keys: dartlab-collect-state-docs- | |
| # HF seed: cold/stale cache 대비 (idempotent). | |
| - name: Seed missing parquet from HF | |
| run: uv run python -X utf8 .github/scripts/sync/seedFromHf.py --category docs | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| - name: Sync recent (docs) | |
| run: uv run python -X utf8 -u .github/scripts/sync/syncRecent.py | |
| env: | |
| DART_API_KEYS: ${{ secrets.DART_API_KEYS }} | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} # checkpoint subprocess (uploadData.py) 가 필요 | |
| SYNC_LOOKBACK_DAYS: ${{ github.event.inputs.lookback_days || '14' }} | |
| SYNC_CATEGORIES: docs | |
| SYNC_STATE_SCOPE: docs | |
| PYTHONUNBUFFERED: "1" | |
| # Phase A — IPC mirror 빌드 | |
| - name: Build IPC mirror (docs) | |
| run: uv run python -X utf8 .github/scripts/sync/buildIpcMirror.py | |
| env: | |
| SYNC_CATEGORY: docs | |
| - name: Upload to HuggingFace (docs) | |
| if: hashFiles('dist/changed_docs.txt') != '' | |
| run: | | |
| if [ -s dist/changed_docs.txt ]; then | |
| cp dist/changed_docs.txt dist/changed.txt | |
| uv run python .github/scripts/sync/uploadData.py --target hf | |
| fi | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| SYNC_CATEGORY: docs | |
| # plan snazzy-wibbling-origami PR-4b — sections SSOT artifact 빌드 + HF push. | |
| # 변경 종목 (dist/changed_docs.txt) 만 buildSections — 종목당 ~18s, 일반 일일 sync | |
| # 의 changed docs 수가 적어 부담 ↓ (수 종목). buildSections 가 dist/changed_sections.txt | |
| # 작성 → uploadData 가 sections category nested 양식으로 HF push. | |
| - name: Build sections artifact from changed docs | |
| if: hashFiles('dist/changed_docs.txt') != '' | |
| run: | | |
| if [ -s dist/changed_docs.txt ]; then | |
| uv run python -X utf8 .github/scripts/sync/buildSections.py | |
| fi | |
| - name: Upload sections to HuggingFace | |
| if: hashFiles('dist/changed_sections.txt') != '' | |
| run: uv run python .github/scripts/sync/uploadData.py --target hf | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| SYNC_CATEGORY: sections | |
| # GitHub Releases 업로드 폐지 (2026-04-08): HF만 유지. | |
| - name: Save docs cache | |
| if: always() | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: data/dart/docs | |
| key: dartlab-data-docs-${{ github.run_id }} | |
| - name: Save collect state — docs scope | |
| if: always() | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: data/dart/_collect_state/docs | |
| key: dartlab-collect-state-docs-${{ github.run_id }} | |
| # ── Job 3 (workflow_dispatch full 전용): 88분기 차집합 (heavy) ── | |
| full-collect: | |
| if: ${{ github.event.inputs.mode == 'full' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 300 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| category: ${{ github.event_name == 'workflow_dispatch' && fromJSON(format('["{0}"]', inputs.category == 'all' && 'finance","report","docs' || inputs.category)) || fromJSON('["finance","report","docs"]') }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Set up Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync | |
| - name: Smoke import (fail fast on src regression) | |
| run: uv run python -c "import dartlab; print('dartlab', getattr(dartlab, '__version__', 'dev'))" | |
| - name: Restore data cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: data/dart/${{ matrix.category }} | |
| key: dartlab-data-${{ matrix.category }}-${{ github.run_id }} | |
| restore-keys: dartlab-data-${{ matrix.category }}- | |
| - name: Full collect (88-quarter diff) | |
| run: uv run python .github/scripts/sync/syncData.py | |
| env: | |
| DART_API_KEYS: ${{ secrets.DART_API_KEYS }} | |
| SYNC_CATEGORY: ${{ matrix.category }} | |
| SYNC_MODE: all | |
| - name: Save data cache | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: data/dart/${{ matrix.category }} | |
| key: dartlab-data-${{ matrix.category }}-${{ github.run_id }} | |
| # Phase A — IPC mirror 빌드 | |
| - name: Build IPC mirror | |
| run: uv run python -X utf8 .github/scripts/sync/buildIpcMirror.py | |
| env: | |
| SYNC_CATEGORY: ${{ matrix.category }} | |
| - name: Upload to HuggingFace | |
| run: uv run python .github/scripts/sync/uploadData.py --target hf | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| SYNC_CATEGORY: ${{ matrix.category }} | |
| # GitHub Releases 업로드 폐지 (2026-04-08): HF만 유지. |