Update Security KG Dataset #40
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Security KG Dataset | |
| on: | |
| schedule: | |
| # Run weekly on Monday at 06:00 UTC | |
| - cron: "0 6 * * 1" | |
| workflow_dispatch: | |
| inputs: | |
| force: | |
| description: "Force reconversion of all sources (ignores fingerprints)" | |
| type: boolean | |
| default: false | |
| sources: | |
| description: "Space-separated sources to convert (empty = auto-detect changed)" | |
| type: string | |
| default: "" | |
| domains: | |
| description: "ATT&CK domains (space-separated)" | |
| type: string | |
| default: "enterprise mobile ics" | |
| parquet_format: | |
| description: "Parquet output format" | |
| type: choice | |
| options: | |
| - v2 | |
| - v1 | |
| default: "v2" | |
| parallel: | |
| description: "Run source conversions in parallel" | |
| type: boolean | |
| default: true | |
| workers: | |
| description: "Number of parallel workers" | |
| type: string | |
| default: "4" | |
| limit: | |
| description: "Limit each source to N triples (0 = no limit)" | |
| type: string | |
| default: "0" | |
| no_combined: | |
| description: "Skip generating combined.parquet" | |
| type: boolean | |
| default: false | |
| no_stats: | |
| description: "Skip generating dashboard stats" | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| update: | |
| runs-on: ubuntu-latest | |
| env: | |
| PYTHONPATH: src | |
| GITHUB_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Check authorization | |
| if: github.event_name != 'schedule' && github.triggering_actor != github.repository_owner | |
| run: | | |
| echo "::error::Only the project admin can manually trigger this workflow." | |
| exit 1 | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.13" | |
| cache: "pip" | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt huggingface_hub | |
| - name: Download metadata from HuggingFace | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| python - <<'PYEOF' | |
| import os, shutil | |
| from pathlib import Path | |
| from huggingface_hub import hf_hub_download | |
| repo_id = "s0u9ata/security-kg" | |
| try: | |
| cached = hf_hub_download( | |
| repo_id=repo_id, | |
| filename=".metadata.json", | |
| repo_type="dataset", | |
| token=os.environ["HF_TOKEN"], | |
| ) | |
| shutil.copy(cached, "hf_dataset/.metadata.json") | |
| print("Downloaded .metadata.json from HuggingFace") | |
| except Exception as e: | |
| print(f"Could not download .metadata.json: {e}") | |
| print("Will process all sources (first run or missing metadata)") | |
| PYEOF | |
| - name: Check which sources have changed | |
| id: source_check | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| FORCE_ALL: ${{ inputs.force }} | |
| SOURCES_OVERRIDE: ${{ inputs.sources }} | |
| run: | | |
| python - <<'PYEOF' | |
| import logging, os | |
| from pathlib import Path | |
| from common import ( | |
| get_all_remote_fingerprints, | |
| load_metadata, | |
| save_metadata, | |
| SOURCE_FINGERPRINT_METHODS, | |
| ) | |
| logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") | |
| previous = load_metadata() | |
| if previous: | |
| print(f"Previous metadata loaded ({len(previous)} keys)") | |
| else: | |
| print("No previous metadata found, will process all sources") | |
| previous_fp = previous.get("source_fingerprints", {}) | |
| # --- Collect current remote fingerprints --- | |
| print("\nFetching remote fingerprints ...") | |
| current_fp = get_all_remote_fingerprints() | |
| # --- Compare each source --- | |
| force = os.environ.get("FORCE_ALL", "false") == "true" | |
| sources_override = os.environ.get("SOURCES_OVERRIDE", "").strip() | |
| override_set = set(sources_override.split()) if sources_override else set() | |
| if force: | |
| print("FORCE MODE: reconverting all sources regardless of fingerprints") | |
| if override_set: | |
| print(f"SOURCES OVERRIDE: only converting {', '.join(sorted(override_set))}") | |
| sources = {} | |
| for source in SOURCE_FINGERPRINT_METHODS: | |
| if force: | |
| is_changed = True | |
| elif override_set: | |
| is_changed = source in override_set | |
| else: | |
| cur = current_fp.get(source, "") | |
| prev = previous_fp.get(source, "") | |
| is_changed = not cur or cur != prev | |
| sources[source] = is_changed | |
| any_changed = any(sources.values()) | |
| with open(os.environ["GITHUB_OUTPUT"], "a") as f: | |
| f.write(f"any_changed={'true' if any_changed else 'false'}\n") | |
| for src, changed in sources.items(): | |
| f.write(f"{src}_changed={'true' if changed else 'false'}\n") | |
| # Save updated metadata (preserve converter fingerprints from previous run) | |
| updated = {"source_fingerprints": current_fp} | |
| if "converter_fingerprints" in previous: | |
| updated["converter_fingerprints"] = previous["converter_fingerprints"] | |
| save_metadata(updated) | |
| print() | |
| for src, changed in sources.items(): | |
| fp = current_fp.get(src, "(failed)") | |
| print(f" {src.upper():>14s}: {'CHANGED' if changed else 'unchanged'} — {fp}") | |
| if not any_changed: | |
| print("\nAll sources unchanged, skipping.") | |
| PYEOF | |
| - name: Download unchanged parquets from HuggingFace | |
| if: steps.source_check.outputs.any_changed == 'true' | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| ATTACK_CHANGED: ${{ steps.source_check.outputs.attack_changed }} | |
| CAPEC_CHANGED: ${{ steps.source_check.outputs.capec_changed }} | |
| CWE_CHANGED: ${{ steps.source_check.outputs.cwe_changed }} | |
| CVE_CHANGED: ${{ steps.source_check.outputs.cve_changed }} | |
| CPE_CHANGED: ${{ steps.source_check.outputs.cpe_changed }} | |
| D3FEND_CHANGED: ${{ steps.source_check.outputs.d3fend_changed }} | |
| ATLAS_CHANGED: ${{ steps.source_check.outputs.atlas_changed }} | |
| CAR_CHANGED: ${{ steps.source_check.outputs.car_changed }} | |
| ENGAGE_CHANGED: ${{ steps.source_check.outputs.engage_changed }} | |
| F3_CHANGED: ${{ steps.source_check.outputs.f3_changed }} | |
| EPSS_CHANGED: ${{ steps.source_check.outputs.epss_changed }} | |
| KEV_CHANGED: ${{ steps.source_check.outputs.kev_changed }} | |
| VULNRICHMENT_CHANGED: ${{ steps.source_check.outputs.vulnrichment_changed }} | |
| GHSA_CHANGED: ${{ steps.source_check.outputs.ghsa_changed }} | |
| SIGMA_CHANGED: ${{ steps.source_check.outputs.sigma_changed }} | |
| EXPLOITDB_CHANGED: ${{ steps.source_check.outputs.exploitdb_changed }} | |
| MISP_GALAXY_CHANGED: ${{ steps.source_check.outputs.misp_galaxy_changed }} | |
| LOLBAS_CHANGED: ${{ steps.source_check.outputs.lolbas_changed }} | |
| LOLDRIVERS_CHANGED: ${{ steps.source_check.outputs.loldrivers_changed }} | |
| ATOMIC_CHANGED: ${{ steps.source_check.outputs.atomic_changed }} | |
| NIST_800_53_CHANGED: ${{ steps.source_check.outputs.nist_800_53_changed }} | |
| NUCLEI_CHANGED: ${{ steps.source_check.outputs.nuclei_changed }} | |
| EUVD_CHANGED: ${{ steps.source_check.outputs.euvd_changed }} | |
| OSV_CHANGED: ${{ steps.source_check.outputs.osv_changed }} | |
| run: | | |
| python - <<'PYEOF' | |
| import os, shutil | |
| from pathlib import Path | |
| from huggingface_hub import hf_hub_download | |
| repo_id = "s0u9ata/security-kg" | |
| output = Path("output") | |
| output.mkdir(exist_ok=True) | |
| # Map source change flags to their parquet files | |
| source_files = { | |
| "attack": ["enterprise", "mobile", "ics", "attack-all"], | |
| "capec": ["capec"], | |
| "cwe": ["cwe"], | |
| "cve": ["cve"], | |
| "cpe": ["cpe"], | |
| "d3fend": ["d3fend"], | |
| "atlas": ["atlas"], | |
| "car": ["car"], | |
| "engage": ["engage"], | |
| "f3": ["f3"], | |
| "epss": ["epss"], | |
| "kev": ["kev"], | |
| "vulnrichment": ["vulnrichment"], | |
| "ghsa": ["ghsa"], | |
| "sigma": ["sigma"], | |
| "exploitdb": ["exploitdb"], | |
| "misp_galaxy": ["misp_galaxy"], | |
| "lolbas": ["lolbas"], | |
| "loldrivers": ["loldrivers"], | |
| "atomic": ["atomic"], | |
| "nist_800_53": ["nist_800_53"], | |
| "nuclei": ["nuclei"], | |
| "euvd": ["euvd"], | |
| "osv": ["osv"], | |
| } | |
| for source, files in source_files.items(): | |
| env_key = f"{source.upper()}_CHANGED" | |
| if os.environ.get(env_key) == "true": | |
| continue | |
| for name in files: | |
| try: | |
| cached = hf_hub_download( | |
| repo_id=repo_id, | |
| filename=f"data/{name}.parquet", | |
| repo_type="dataset", | |
| token=os.environ["HF_TOKEN"], | |
| ) | |
| shutil.copy(cached, output / f"{name}.parquet") | |
| print(f"Downloaded {name}.parquet (unchanged)") | |
| except Exception as e: | |
| print(f"Could not download {name}.parquet: {e}") | |
| print("Will regenerate from source") | |
| PYEOF | |
| - name: Convert changed sources | |
| if: steps.source_check.outputs.any_changed == 'true' | |
| timeout-minutes: 90 | |
| run: | | |
| # Build --sources list from changed flags | |
| sources="" | |
| for src in attack capec cwe cve cpe d3fend atlas car engage f3 epss kev vulnrichment ghsa sigma exploitdb misp_galaxy lolbas loldrivers atomic nist_800_53 nuclei euvd osv; do | |
| case "$src" in | |
| attack) [ "${{ steps.source_check.outputs.attack_changed }}" = "true" ] && sources="$sources attack" ;; | |
| capec) [ "${{ steps.source_check.outputs.capec_changed }}" = "true" ] && sources="$sources capec" ;; | |
| cwe) [ "${{ steps.source_check.outputs.cwe_changed }}" = "true" ] && sources="$sources cwe" ;; | |
| cve) [ "${{ steps.source_check.outputs.cve_changed }}" = "true" ] && sources="$sources cve" ;; | |
| cpe) [ "${{ steps.source_check.outputs.cpe_changed }}" = "true" ] && sources="$sources cpe" ;; | |
| d3fend) [ "${{ steps.source_check.outputs.d3fend_changed }}" = "true" ] && sources="$sources d3fend" ;; | |
| atlas) [ "${{ steps.source_check.outputs.atlas_changed }}" = "true" ] && sources="$sources atlas" ;; | |
| car) [ "${{ steps.source_check.outputs.car_changed }}" = "true" ] && sources="$sources car" ;; | |
| engage) [ "${{ steps.source_check.outputs.engage_changed }}" = "true" ] && sources="$sources engage" ;; | |
| f3) [ "${{ steps.source_check.outputs.f3_changed }}" = "true" ] && sources="$sources f3" ;; | |
| epss) [ "${{ steps.source_check.outputs.epss_changed }}" = "true" ] && sources="$sources epss" ;; | |
| kev) [ "${{ steps.source_check.outputs.kev_changed }}" = "true" ] && sources="$sources kev" ;; | |
| vulnrichment) [ "${{ steps.source_check.outputs.vulnrichment_changed }}" = "true" ] && sources="$sources vulnrichment" ;; | |
| ghsa) [ "${{ steps.source_check.outputs.ghsa_changed }}" = "true" ] && sources="$sources ghsa" ;; | |
| sigma) [ "${{ steps.source_check.outputs.sigma_changed }}" = "true" ] && sources="$sources sigma" ;; | |
| exploitdb) [ "${{ steps.source_check.outputs.exploitdb_changed }}" = "true" ] && sources="$sources exploitdb" ;; | |
| misp_galaxy) [ "${{ steps.source_check.outputs.misp_galaxy_changed }}" = "true" ] && sources="$sources misp_galaxy" ;; | |
| lolbas) [ "${{ steps.source_check.outputs.lolbas_changed }}" = "true" ] && sources="$sources lolbas" ;; | |
| loldrivers) [ "${{ steps.source_check.outputs.loldrivers_changed }}" = "true" ] && sources="$sources loldrivers" ;; | |
| atomic) [ "${{ steps.source_check.outputs.atomic_changed }}" = "true" ] && sources="$sources atomic" ;; | |
| nist_800_53) [ "${{ steps.source_check.outputs.nist_800_53_changed }}" = "true" ] && sources="$sources nist_800_53" ;; | |
| nuclei) [ "${{ steps.source_check.outputs.nuclei_changed }}" = "true" ] && sources="$sources nuclei" ;; | |
| euvd) [ "${{ steps.source_check.outputs.euvd_changed }}" = "true" ] && sources="$sources euvd" ;; | |
| osv) [ "${{ steps.source_check.outputs.osv_changed }}" = "true" ] && sources="$sources osv" ;; | |
| esac | |
| done | |
| sources=$(echo $sources | xargs) | |
| echo "Changed sources: $sources" | |
| if [ -z "$sources" ]; then | |
| echo "No sources to convert" | |
| exit 0 | |
| fi | |
| # Build convert.py command from workflow inputs (with defaults for scheduled runs) | |
| PARQUET_FORMAT="${{ inputs.parquet_format }}" | |
| DOMAINS="${{ inputs.domains }}" | |
| PARALLEL="${{ inputs.parallel }}" | |
| WORKERS="${{ inputs.workers }}" | |
| LIMIT="${{ inputs.limit }}" | |
| cmd="python src/convert.py --sources $sources --cache-dir /tmp/source-cache --log-dir logs --no-combined --no-stats" | |
| cmd="$cmd --parquet-format ${PARQUET_FORMAT:-v2}" | |
| cmd="$cmd --domains ${DOMAINS:-enterprise mobile ics}" | |
| if [ "${PARALLEL:-true}" != "false" ]; then | |
| cmd="$cmd --parallel --workers ${WORKERS:-4}" | |
| fi | |
| if [ -n "$LIMIT" ] && [ "$LIMIT" != "0" ]; then | |
| cmd="$cmd --limit $LIMIT" | |
| fi | |
| echo "Running: $cmd" | |
| $cmd | |
| - name: Download fallback parquets for failed sources | |
| if: steps.source_check.outputs.any_changed == 'true' | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| python - <<'PYEOF' | |
| import json, os, shutil, sys | |
| from pathlib import Path | |
| from huggingface_hub import hf_hub_download | |
| report_path = Path("hf_dataset/.conversion_report.json") | |
| if not report_path.exists(): | |
| print("No conversion report found, skipping fallback download") | |
| sys.exit(0) | |
| report = json.loads(report_path.read_text()) | |
| failed = report.get("failed_sources", []) | |
| if not failed: | |
| print("No failed sources, no fallbacks needed") | |
| sys.exit(0) | |
| print(f"Failed sources: {', '.join(failed)}") | |
| repo_id = "s0u9ata/security-kg" | |
| output = Path("output") | |
| source_files = { | |
| "attack": ["enterprise", "mobile", "ics", "attack-all"], | |
| "capec": ["capec"], "cwe": ["cwe"], "cve": ["cve"], "cpe": ["cpe"], | |
| "d3fend": ["d3fend"], "atlas": ["atlas"], "car": ["car"], | |
| "engage": ["engage"], "f3": ["f3"], "epss": ["epss"], "kev": ["kev"], | |
| "vulnrichment": ["vulnrichment"], "ghsa": ["ghsa"], | |
| "sigma": ["sigma"], "exploitdb": ["exploitdb"], | |
| "misp_galaxy": ["misp_galaxy"], | |
| "lolbas": ["lolbas"], "loldrivers": ["loldrivers"], | |
| "atomic": ["atomic"], "nist_800_53": ["nist_800_53"], | |
| "nuclei": ["nuclei"], "euvd": ["euvd"], "osv": ["osv"], | |
| } | |
| for source in failed: | |
| for name in source_files.get(source, [source]): | |
| pq_path = output / f"{name}.parquet" | |
| if pq_path.exists(): | |
| print(f" {name}.parquet already exists, skipping") | |
| continue | |
| try: | |
| cached = hf_hub_download( | |
| repo_id=repo_id, | |
| filename=f"data/{name}.parquet", | |
| repo_type="dataset", | |
| token=os.environ["HF_TOKEN"], | |
| ) | |
| shutil.copy(cached, pq_path) | |
| print(f" Downloaded fallback {name}.parquet from HuggingFace") | |
| except Exception as e: | |
| print(f" ERROR: Could not download fallback {name}.parquet: {e}") | |
| PYEOF | |
| - name: Rebuild combined.parquet from all sources | |
| if: steps.source_check.outputs.any_changed == 'true' && inputs.no_combined != true | |
| env: | |
| PARQUET_FORMAT: ${{ inputs.parquet_format || 'v2' }} | |
| run: | | |
| python - <<'PYEOF' | |
| import os | |
| from pathlib import Path | |
| import pyarrow.parquet as pq | |
| import pyarrow as pa | |
| from common import PARQUET_SCHEMA, PARQUET_FORMATS | |
| parquet_names = [ | |
| "attack-all", "capec", "cwe", "cve", "cpe", | |
| "d3fend", "atlas", "car", "engage", "f3", "epss", "kev", | |
| "vulnrichment", "ghsa", "sigma", "exploitdb", "misp_galaxy", | |
| "lolbas", "loldrivers", "atomic", "nist_800_53", | |
| "nuclei", "euvd", "osv", | |
| ] | |
| output = Path("output") | |
| parquet_files = [output / f"{n}.parquet" for n in parquet_names if (output / f"{n}.parquet").exists()] | |
| if len(parquet_files) > 1: | |
| combined_path = output / "combined.parquet" | |
| pq_opts = PARQUET_FORMATS[os.environ.get("PARQUET_FORMAT", "v2")] | |
| writer = None | |
| total_rows = 0 | |
| try: | |
| for pf in parquet_files: | |
| for batch in pq.ParquetFile(pf).iter_batches(batch_size=500_000): | |
| if writer is None: | |
| writer = pq.ParquetWriter(combined_path, PARQUET_SCHEMA, **pq_opts) | |
| writer.write_table(pa.Table.from_batches([batch], schema=PARQUET_SCHEMA)) | |
| total_rows += batch.num_rows | |
| finally: | |
| if writer is not None: | |
| writer.close() | |
| print(f"Built combined.parquet: {total_rows:,} triples from {len(parquet_files)} files") | |
| PYEOF | |
| - name: Validate generated data | |
| if: steps.source_check.outputs.any_changed == 'true' | |
| run: | | |
| python - <<'PYEOF' | |
| import json, sys | |
| import pyarrow.parquet as pq | |
| from pathlib import Path | |
| # Load conversion report to identify fallback sources | |
| report_path = Path("hf_dataset/.conversion_report.json") | |
| failed_sources = [] | |
| if report_path.exists(): | |
| report = json.loads(report_path.read_text()) | |
| failed_sources = report.get("failed_sources", []) | |
| source_to_parquets = { | |
| "attack": ["enterprise", "mobile", "ics", "attack-all"], | |
| "capec": ["capec"], "cwe": ["cwe"], "cve": ["cve"], "cpe": ["cpe"], | |
| "d3fend": ["d3fend"], "atlas": ["atlas"], "car": ["car"], | |
| "engage": ["engage"], "f3": ["f3"], "epss": ["epss"], "kev": ["kev"], | |
| "vulnrichment": ["vulnrichment"], "ghsa": ["ghsa"], | |
| "sigma": ["sigma"], "exploitdb": ["exploitdb"], | |
| "misp_galaxy": ["misp_galaxy"], | |
| "lolbas": ["lolbas"], "loldrivers": ["loldrivers"], | |
| "atomic": ["atomic"], "nist_800_53": ["nist_800_53"], | |
| "nuclei": ["nuclei"], "euvd": ["euvd"], "osv": ["osv"], | |
| } | |
| fallback_parquets = set() | |
| for src in failed_sources: | |
| for pq_name in source_to_parquets.get(src, [src]): | |
| fallback_parquets.add(pq_name) | |
| MIN_TRIPLES = { | |
| "enterprise": 20_000, "mobile": 2_000, "ics": 1_500, | |
| "attack-all": 25_000, "capec": 4_000, "cwe": 7_000, | |
| "cve": 500_000, "cpe": 500_000, "d3fend": 500, | |
| "atlas": 100, "car": 100, "engage": 100, "f3": 100, | |
| "epss": 100_000, "kev": 2_000, "vulnrichment": 100_000, | |
| "ghsa": 50_000, "sigma": 10_000, "exploitdb": 50_000, | |
| "misp_galaxy": 50_000, | |
| "lolbas": 1_000, "loldrivers": 2_000, | |
| "atomic": 5_000, "nist_800_53": 1_000, | |
| "nuclei": 10_000, "euvd": 1_000, "osv": 100_000, | |
| "combined": 1_000_000, | |
| } | |
| failed = False | |
| for name, minimum in MIN_TRIPLES.items(): | |
| path = Path(f"output/{name}.parquet") | |
| if not path.exists(): | |
| if name in fallback_parquets: | |
| print(f"FAIL: {name}.parquet missing (conversion failed + no fallback)") | |
| failed = True | |
| else: | |
| print(f"WARN: {path} does not exist (source may not have changed)") | |
| continue | |
| count = pq.read_metadata(path).num_rows | |
| size_kb = path.stat().st_size / 1024 | |
| if name in fallback_parquets: | |
| print(f"FALLBACK: {name}.parquet — {count:,} triples, {size_kb:.0f} KB (last good version)") | |
| elif count < minimum: | |
| print(f"FAIL: {name}.parquet has {count} triples (minimum: {minimum})") | |
| failed = True | |
| else: | |
| print(f"OK: {name}.parquet — {count:,} triples, {size_kb:.0f} KB") | |
| if failed: | |
| print("\nValidation failed — aborting to protect existing HuggingFace data.") | |
| sys.exit(1) | |
| if failed_sources: | |
| print(f"\nNote: {len(failed_sources)} source(s) used fallback: {', '.join(failed_sources)}") | |
| PYEOF | |
| - name: Update dataset card with fresh counts and date | |
| if: steps.source_check.outputs.any_changed == 'true' | |
| run: | | |
| python - <<'PYEOF' | |
| import json | |
| from pathlib import Path | |
| from common import update_dataset_readme | |
| report_path = Path("hf_dataset/.conversion_report.json") | |
| failed_sources = [] | |
| if report_path.exists(): | |
| report = json.loads(report_path.read_text()) | |
| failed_sources = report.get("failed_sources", []) | |
| update_dataset_readme(Path("output"), failed_sources=failed_sources) | |
| PYEOF | |
| - name: Generate dashboard stats | |
| if: steps.source_check.outputs.any_changed == 'true' && inputs.no_stats != true | |
| run: | | |
| python src/generate_stats.py --output-dir output --stats-dir hf_dataset/.stats | |
| - name: Pre-compute top-N entity neighborhoods | |
| if: steps.source_check.outputs.any_changed == 'true' && inputs.no_stats != true | |
| run: | | |
| python src/generate_neighborhoods.py \ | |
| --output-dir output \ | |
| --neighborhoods-dir hf_dataset/.neighborhoods \ | |
| --parquet combined.parquet | |
| - name: Upload to HuggingFace Hub | |
| if: steps.source_check.outputs.any_changed == 'true' | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| ATTACK_CHANGED: ${{ steps.source_check.outputs.attack_changed }} | |
| CAPEC_CHANGED: ${{ steps.source_check.outputs.capec_changed }} | |
| CWE_CHANGED: ${{ steps.source_check.outputs.cwe_changed }} | |
| CVE_CHANGED: ${{ steps.source_check.outputs.cve_changed }} | |
| CPE_CHANGED: ${{ steps.source_check.outputs.cpe_changed }} | |
| D3FEND_CHANGED: ${{ steps.source_check.outputs.d3fend_changed }} | |
| ATLAS_CHANGED: ${{ steps.source_check.outputs.atlas_changed }} | |
| CAR_CHANGED: ${{ steps.source_check.outputs.car_changed }} | |
| ENGAGE_CHANGED: ${{ steps.source_check.outputs.engage_changed }} | |
| F3_CHANGED: ${{ steps.source_check.outputs.f3_changed }} | |
| EPSS_CHANGED: ${{ steps.source_check.outputs.epss_changed }} | |
| KEV_CHANGED: ${{ steps.source_check.outputs.kev_changed }} | |
| VULNRICHMENT_CHANGED: ${{ steps.source_check.outputs.vulnrichment_changed }} | |
| GHSA_CHANGED: ${{ steps.source_check.outputs.ghsa_changed }} | |
| SIGMA_CHANGED: ${{ steps.source_check.outputs.sigma_changed }} | |
| EXPLOITDB_CHANGED: ${{ steps.source_check.outputs.exploitdb_changed }} | |
| MISP_GALAXY_CHANGED: ${{ steps.source_check.outputs.misp_galaxy_changed }} | |
| LOLBAS_CHANGED: ${{ steps.source_check.outputs.lolbas_changed }} | |
| LOLDRIVERS_CHANGED: ${{ steps.source_check.outputs.loldrivers_changed }} | |
| ATOMIC_CHANGED: ${{ steps.source_check.outputs.atomic_changed }} | |
| NIST_800_53_CHANGED: ${{ steps.source_check.outputs.nist_800_53_changed }} | |
| NUCLEI_CHANGED: ${{ steps.source_check.outputs.nuclei_changed }} | |
| EUVD_CHANGED: ${{ steps.source_check.outputs.euvd_changed }} | |
| OSV_CHANGED: ${{ steps.source_check.outputs.osv_changed }} | |
| run: | | |
| python - <<'PYEOF' | |
| import json, os, shutil | |
| from pathlib import Path | |
| from huggingface_hub import HfApi | |
| repo_id = "s0u9ata/security-kg" | |
| api = HfApi(token=os.environ["HF_TOKEN"]) | |
| # Ensure the dataset repo exists | |
| api.create_repo(repo_id=repo_id, repo_type="dataset", exist_ok=True) | |
| # Only upload parquets for changed sources (unchanged ones are already on HF) | |
| source_files = { | |
| "attack": ["enterprise", "mobile", "ics", "attack-all"], | |
| "capec": ["capec"], "cwe": ["cwe"], "cve": ["cve"], "cpe": ["cpe"], | |
| "d3fend": ["d3fend"], "atlas": ["atlas"], "car": ["car"], | |
| "engage": ["engage"], "f3": ["f3"], "epss": ["epss"], "kev": ["kev"], | |
| "vulnrichment": ["vulnrichment"], "ghsa": ["ghsa"], | |
| "sigma": ["sigma"], "exploitdb": ["exploitdb"], | |
| "misp_galaxy": ["misp_galaxy"], | |
| "lolbas": ["lolbas"], "loldrivers": ["loldrivers"], | |
| "atomic": ["atomic"], "nist_800_53": ["nist_800_53"], | |
| "nuclei": ["nuclei"], "euvd": ["euvd"], "osv": ["osv"], | |
| } | |
| # Identify which parquets actually need uploading | |
| changed_parquets = set() | |
| for source, files in source_files.items(): | |
| if os.environ.get(f"{source.upper()}_CHANGED") == "true": | |
| changed_parquets.update(files) | |
| # combined.parquet is always rebuilt when any source changes | |
| changed_parquets.add("combined") | |
| staging = Path("hf_staging") | |
| staging.mkdir(exist_ok=True) | |
| (staging / "data").mkdir(exist_ok=True) | |
| for name in changed_parquets: | |
| src = Path(f"output/{name}.parquet") | |
| if src.exists(): | |
| shutil.copy(src, staging / "data" / f"{name}.parquet") | |
| print(f"Staging {name}.parquet (changed)") | |
| # Copy stats JSON files | |
| stats_src = Path("hf_dataset/.stats") | |
| if stats_src.is_dir(): | |
| stats_dst = staging / "stats" | |
| stats_dst.mkdir(exist_ok=True) | |
| for sf in stats_src.glob("*.stats.json"): | |
| shutil.copy(sf, stats_dst / sf.name) | |
| print(f"Staging {sf.name}") | |
| # Copy pre-computed neighborhood JSONs (top-N hot entities + index) | |
| neigh_src = Path("hf_dataset/.neighborhoods") | |
| if neigh_src.is_dir(): | |
| neigh_dst = staging / "neighborhoods" | |
| neigh_dst.mkdir(exist_ok=True) | |
| for nf in neigh_src.glob("*.json"): | |
| shutil.copy(nf, neigh_dst / nf.name) | |
| print(f"Staging {sum(1 for _ in neigh_dst.glob('*.json'))} neighborhood JSONs") | |
| shutil.copy("hf_dataset/README.md", staging / "README.md") | |
| shutil.copy("hf_dataset/.metadata.json", staging / ".metadata.json") | |
| report = Path("hf_dataset/.conversion_report.json") | |
| if report.exists(): | |
| shutil.copy(report, staging / ".conversion_report.json") | |
| # Upload only changed files in one commit (upload_folder does not | |
| # delete remote files that are absent from the local staging dir) | |
| api.upload_folder( | |
| folder_path=str(staging), | |
| repo_id=repo_id, | |
| repo_type="dataset", | |
| commit_message="Update dataset", | |
| ) | |
| print("Uploaded changed files to HuggingFace Hub.") | |
| PYEOF | |
| - name: Open PR to update dataset card counts | |
| if: steps.source_check.outputs.any_changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| git diff --quiet hf_dataset/README.md README.md && exit 0 | |
| branch="update-dataset-card-counts-${{ github.run_id }}" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git checkout -b "$branch" | |
| git add hf_dataset/README.md hf_dataset/.metadata.json hf_dataset/.conversion_report.json hf_dataset/.stats/ hf_dataset/.neighborhoods/ README.md | |
| git commit -m "Update dataset counts" | |
| git push -u origin "$branch" | |
| existing=$(gh pr list --head "$branch" --state open --json number --jq '.[0].number') | |
| if [ -z "$existing" ]; then | |
| gh pr create \ | |
| --title "Update dataset card counts" \ | |
| --body "Automated update of triple counts and date in the HuggingFace dataset card." \ | |
| --base main | |
| fi | |
| - name: Upload conversion logs | |
| if: always() && steps.source_check.outputs.any_changed == 'true' | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: conversion-logs | |
| path: logs/ | |
| retention-days: 30 | |
| - name: Log summary | |
| if: always() && steps.source_check.outputs.any_changed == 'true' | |
| run: | | |
| run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| echo "Conversion logs available at: ${run_url}" | |
| echo "Download logs artifact from the workflow run page." | |
| if [ -d logs ]; then | |
| echo "" | |
| echo "--- Log Summary ---" | |
| grep -rh -E '(Starting|Finished|Wrote|FAIL|ERROR|All done)' logs/*.log 2>/dev/null | tail -50 | |
| fi |