Data Prebuild (DART) #224
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Data Prebuild (DART) | |
| # ────────────────────────────────────────────────────────────── | |
| # DART Data Sync 완료 후 자동 트리거 (증분) + 주 1 회 full 재생성. | |
| # scan 프리빌드(changes/finance/report/sharesOutstanding/docsIndex) 생성 → HF 업로드. | |
| # | |
| # 증분(기본): panel 변경 종목만 다운로드 → changes/sharesOutstanding/docsIndex 변경분만 | |
| # 재계산 후 기존 parquet 에 종목 단위 머지. 전 92K panel(11GB) seed 금지 = OOM/디스크 | |
| # 고갈 근본원인 제거. finance/report 는 캐시 full 입력이라 full 빌드. | |
| # full(일요일 cron / dispatch full=true): 디스크 확보 후 전 종목 panel seed → 전체 재생성. | |
| # 빌더 변경·backfill drift 교정 안전망. _scanBuildState ledger 가 두 경로를 잇는다. | |
| # | |
| # 입력 seed 는 prebuildData.py 가 단일 SSOT (BASE_SEED_CATEGORIES). EDGAR 는 edgarSync.yml 전담. | |
| # ────────────────────────────────────────────────────────────── | |
| on: | |
| workflow_run: | |
| workflows: ["Data Sync"] | |
| types: [completed] | |
| schedule: | |
| - cron: '0 17 * * 0' # UTC 17:00 일 = KST 02:00 월, 주간 full 재생성 (Data Sync 18:00 와 1h 이격) | |
| workflow_dispatch: | |
| inputs: | |
| full: | |
| description: '전 종목 full 재생성 (panel 전량 seed, 디스크 확보)' | |
| default: 'false' | |
| type: choice | |
| options: ['false', 'true'] | |
| concurrency: | |
| group: hf-dart-push | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read # HF push 는 HF_TOKEN 사용 — git 작업 없음(checkout+cache 만). 최소권한. | |
| jobs: | |
| # ── 증분: Data Sync 직후 / dispatch(full=false). panel 변경분만 ── | |
| prebuild-scan: | |
| if: ${{ (github.event_name == 'workflow_run' && (github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.full != 'true') }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| # incremental 도 finance/report/scan 캐시 + 임시 parquet merge 가 동시에 생긴다. | |
| # 기본 runner 디스크가 빌드 중 고갈되면 Python 예외 없이 runner 통신 상실로 끝난다. | |
| - name: Free disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL || true | |
| df -h / | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| - name: Set up Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync | |
| - name: Smoke import (fail fast on src regression) | |
| run: uv run python -c "import dartlab; print('dartlab', getattr(dartlab, '__version__', 'dev'))" | |
| # workflow_run 컨텍스트는 소스 run_id 를 모름 → restore-keys 접두사 매칭으로 최신 캐시. | |
| - name: Restore finance cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: data/dart/finance | |
| key: dartlab-data-finance-prebuild-never-match | |
| restore-keys: dartlab-data-finance- | |
| - name: Restore report cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: data/dart/report | |
| key: dartlab-data-report-prebuild-never-match | |
| restore-keys: dartlab-data-report- | |
| # 직전 scan 산출물(+_scanBuildState ledger) 복원 → 증분 머지/prune 기준. cache miss 면 | |
| # prebuildData 의 base seed 가 HF 에서 scan 카테고리를 idempotent 충전한다. | |
| - name: Restore scan cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: data/dart/scan | |
| key: dartlab-data-scan-prebuild-never-match | |
| restore-keys: dartlab-data-scan- | |
| # KIND/dartList/corpProfile seed 는 prebuildData.py 가 retryHfCall SSOT 로 자체 수행 | |
| # (offline 가드 아래 HF 다운로드 — buildScan 의 KIND OfflineViolation 회피). | |
| - name: Build DART scan prebuilds (incremental) + upload | |
| run: | | |
| echo "[prebuild] start incremental" | |
| df -h / | |
| free -h || true | |
| uv run python -u -X utf8 .github/scripts/prebuild/prebuildData.py & | |
| pid=$! | |
| ( | |
| elapsed=0 | |
| while kill -0 "$pid" 2>/dev/null; do | |
| sleep 60 | |
| elapsed=$((elapsed + 60)) | |
| echo "[prebuild] heartbeat incremental elapsed=${elapsed}s" | |
| df -h / | |
| free -h || true | |
| done | |
| ) & | |
| heartbeat=$! | |
| set +e | |
| wait "$pid" | |
| status=$? | |
| set -e | |
| echo "[prebuild] child incremental exit=${status}" | |
| df -h / | |
| free -h || true | |
| kill "$heartbeat" 2>/dev/null || true | |
| exit "$status" | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| PYTHONUNBUFFERED: "1" | |
| DARTLAB_SCAN_BATCH_SIZE: '50' | |
| POLARS_MAX_THREADS: '2' | |
| MALLOC_ARENA_MAX: '2' | |
| DARTLAB_HF_RETRY_ATTEMPTS: '3' | |
| DARTLAB_HF_RETRY_MAX_SINGLE_WAIT_SECONDS: '120' | |
| - name: Save DART scan cache | |
| if: always() | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: data/dart/scan | |
| key: dartlab-data-scan-${{ github.run_id }} | |
| # ── full: 주간 / dispatch(full=true). 전 종목 panel seed (디스크 확보) ── | |
| prebuild-full: | |
| if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.full == 'true') }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| # 전 종목 panel(~11GB) seed 를 위한 디스크 확보 — 기본 러너 / 파티션 ~14GB 한계 회피. | |
| - name: Free disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL || true | |
| df -h / | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| - name: Set up Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync | |
| - name: Smoke import (fail fast on src regression) | |
| run: uv run python -c "import dartlab; print('dartlab', getattr(dartlab, '__version__', 'dev'))" | |
| - name: Restore finance cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: data/dart/finance | |
| key: dartlab-data-finance-prebuild-never-match | |
| restore-keys: dartlab-data-finance- | |
| - name: Restore report cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: data/dart/report | |
| key: dartlab-data-report-prebuild-never-match | |
| restore-keys: dartlab-data-report- | |
| # KIND/dartList/corpProfile seed 는 prebuildData.py 가 retryHfCall SSOT 로 자체 수행. | |
| - name: Build DART scan prebuilds (full rebuild) + upload | |
| run: | | |
| echo "[prebuild] start full" | |
| df -h / | |
| free -h || true | |
| uv run python -u -X utf8 .github/scripts/prebuild/prebuildData.py & | |
| pid=$! | |
| ( | |
| elapsed=0 | |
| while kill -0 "$pid" 2>/dev/null; do | |
| sleep 60 | |
| elapsed=$((elapsed + 60)) | |
| echo "[prebuild] heartbeat full elapsed=${elapsed}s" | |
| df -h / | |
| free -h || true | |
| done | |
| ) & | |
| heartbeat=$! | |
| set +e | |
| wait "$pid" | |
| status=$? | |
| set -e | |
| echo "[prebuild] child full exit=${status}" | |
| df -h / | |
| free -h || true | |
| kill "$heartbeat" 2>/dev/null || true | |
| exit "$status" | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| PREBUILD_FULL: "1" | |
| PYTHONUNBUFFERED: "1" | |
| DARTLAB_SCAN_BATCH_SIZE: '50' | |
| POLARS_MAX_THREADS: '2' | |
| MALLOC_ARENA_MAX: '2' | |
| DARTLAB_HF_RETRY_ATTEMPTS: '3' | |
| DARTLAB_HF_RETRY_MAX_SINGLE_WAIT_SECONDS: '120' | |
| - name: Save DART scan cache | |
| if: always() | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: data/dart/scan | |
| key: dartlab-data-scan-${{ github.run_id }} |