feat(ml,mlebench): switch mlebench to OOF-driven eval and improve score workflow

agents/ml_agent/evocoder/evaluator.py

@@ -758,7 +758,13 @@ def evaluate(temp_dir):
         assert isinstance(result, dict), "The workflow() function must return a dict."
 
         # check required keys
-        REQUIRED_KEYS = ["submission_file_path", "prediction_stats"]
+        REQUIRED_KEYS = [
+            "submission_file_path",
+            "oof_submission_file_path",
+            "oof_answer_file_path",
+            "oof_coverage",
+            "prediction_stats",
+        ]
         missing_keys = [k for k in REQUIRED_KEYS if k not in result]
         if missing_keys:
             return {{
@@ -767,6 +773,123 @@ def evaluate(temp_dir):
                 "summary": f"Missing required keys in workflow result: {{missing_keys}}",
                 "artifacts": {{"workflow": result}}
             }}
+
+        # check submission files exist
+        submission_file = result.get("submission_file_path")
+        if not os.path.isfile(submission_file):
+            return {{
+                "score": 0.0,
+                "status": "validation_failed",
+                "summary": f"Test submission file not found: {{submission_file}}",
+                "artifacts": {{"workflow": result}}
+            }}
+
+        oof_submission_file = result.get("oof_submission_file_path")
+        if not os.path.isfile(oof_submission_file):
+            return {{
+                "score": 0.0,
+                "status": "validation_failed",
+                "summary": f"OOF submission file not found: {{oof_submission_file}}",
+                "artifacts": {{"workflow": result}}
+            }}
+
+        oof_answer_file = result.get("oof_answer_file_path")
+        if not os.path.isfile(oof_answer_file):
+            return {{
+                "score": 0.0,
+                "status": "validation_failed",
+                "summary": f"OOF answer file not found: {{oof_answer_file}}",
+                "artifacts": {{"workflow": result}}
+            }}
+
+        oof_coverage = result.get("oof_coverage")
+        if isinstance(oof_coverage, bool) or not isinstance(oof_coverage, (int, float)):
+            return {{
+                "score": 0.0,
+                "status": "validation_failed",
+                "summary": f"oof_coverage must be numeric in [0,1], got {{type(oof_coverage).__name__}}",
+                "artifacts": {{"workflow": result}}
+            }}
+        oof_coverage = float(oof_coverage)
+        if oof_coverage < 0.0 or oof_coverage > 1.0:
+            return {{
+                "score": 0.0,
+                "status": "validation_failed",
+                "summary": f"oof_coverage must be in [0,1], got {{oof_coverage}}",
+                "artifacts": {{"workflow": result}}
+            }}
+
+        # csv-level validation
+        try:
+            submission_df = pd.read_csv(submission_file)
+            oof_submission_df = pd.read_csv(oof_submission_file)
+            oof_answer_df = pd.read_csv(oof_answer_file)
+        except Exception as e:
+            return {{
+                "score": 0.0,
+                "status": "validation_failed",
+                "summary": f"Failed to read workflow csv artifacts: {{e}}",
+                "artifacts": {{"workflow": result}}
+            }}
+
+        if submission_df.empty:
+            return {{
+                "score": 0.0,
+                "status": "validation_failed",
+                "summary": "Final submission csv is empty.",
+                "artifacts": {{"workflow": result}}
+            }}
+        if oof_submission_df.empty:
+            return {{
+                "score": 0.0,
+                "status": "validation_failed",
+                "summary": "OOF submission csv is empty.",
+                "artifacts": {{"workflow": result}}
+            }}
+        if oof_answer_df.empty:
+            return {{
+                "score": 0.0,
+                "status": "validation_failed",
+                "summary": "OOF answer csv is empty.",
+                "artifacts": {{"workflow": result}}
+            }}
+
+        # final submission and oof submission should share the same prediction schema
+        submission_cols = list(submission_df.columns)
+        oof_submission_cols = list(oof_submission_df.columns)
+        if submission_cols != oof_submission_cols:
+            return {{
+                "score": 0.0,
+                "status": "validation_failed",
+                "summary": "Column mismatch between final submission and OOF submission.",
+                "artifacts": {{
+                    "workflow": result,
+                    "final_submission_columns": submission_cols,
+                    "oof_submission_columns": oof_submission_cols,
+                }}
+            }}
+
+        # oof submission and oof answer should align row-wise
+        if len(oof_submission_df) != len(oof_answer_df):
+            return {{
+                "score": 0.0,
+                "status": "validation_failed",
+                "summary": f"Row count mismatch: oof_submission={{len(oof_submission_df)}}, oof_answer={{len(oof_answer_df)}}",
+                "artifacts": {{"workflow": result}}
+            }}
+
+        missing_answer_cols = [col for col in oof_submission_cols if col not in oof_answer_df.columns]
+        if missing_answer_cols:
+            return {{
+                "score": 0.0,
+                "status": "validation_failed",
+                "summary": f"oof_answer_file_path is missing required submission columns: {{missing_answer_cols}}",
+                "artifacts": {{
+                    "workflow": result,
+                    "oof_submission_columns": oof_submission_cols,
+                    "oof_answer_columns": list(oof_answer_df.columns),
+                }}
+            }}
         # check prediction_stats structure
         prediction_stats = result["prediction_stats"]
         assert isinstance(prediction_stats, dict), "prediction_stats must be a dict."
@@ -806,12 +929,12 @@ def evaluate(temp_dir):
                         "artifacts": {{"workflow": result}}
                     }}
         return {{
-            "score": 1.0, 
-            "status": "success", 
-            "summary": "workflow validation passed and submission file created.",
+            "score": 1.0,
+            "status": "success",
+            "summary": "workflow validation passed and submission files created.",
             "artifacts": {{"workflow": result}}
         }}
 
     except Exception as e:
         return {{"score": 0.0, "status": "validation_failed", "summary": f"{{e}}", "artifacts": {{"traceback": traceback.format_exc()}}}}
-"""
+"""

agents/ml_agent/examples/environment_cpu.yaml

@@ -1,7 +1,6 @@
 name: loongflow_ml
 channels:
   - conda-forge
-  - pytorch
   - defaults
 
 dependencies:
@@ -15,9 +14,9 @@ dependencies:
   - git
 
   # ==================== 3. PyTorch & RAPIDS ====================
-  - pytorch::pytorch=2.4.0
-  - pytorch::torchvision=0.19.0
-  - pytorch::torchaudio=2.4.0
+  - pytorch=2.4.0
+  - torchvision=0.19.0
+  - torchaudio=2.4.0
 
   - py-xgboost=2.1
   - lightgbm=4.5.0

agents/ml_agent/examples/mlebench/checkpoint_submission_locator.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Locate the latest checkpoint and extract submission_file_path from best_solution.json.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Any
+
+
+def _checkpoint_sort_key(path: Path) -> tuple[int, int, int, float, str]:
+    m = re.search(r"checkpoint-iter-(\d+)-(\d+)$", path.name)
+    mtime = path.stat().st_mtime
+    if m:
+        return (1, int(m.group(1)), int(m.group(2)), mtime, path.name)
+    return (0, 0, 0, mtime, path.name)
+
+
+def find_latest_checkpoint_dir(checkpoints_root: Path) -> Path:
+    if not checkpoints_root.is_dir():
+        raise FileNotFoundError(f"Checkpoint root not found: {checkpoints_root}")
+
+    checkpoint_dirs = [p for p in checkpoints_root.iterdir() if p.is_dir()]
+    if not checkpoint_dirs:
+        raise FileNotFoundError(f"No checkpoint directory found under: {checkpoints_root}")
+
+    checkpoint_dirs.sort(key=_checkpoint_sort_key)
+    return checkpoint_dirs[-1]
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception as exc:
+        raise ValueError(f"Failed to parse JSON: {path}") from exc
+
+
+def extract_submission_path(best_solution_path: Path) -> str:
+    if not best_solution_path.is_file():
+        raise FileNotFoundError(f"best_solution.json not found: {best_solution_path}")
+
+    best_solution = _load_json(best_solution_path)
+    evaluation = best_solution.get("evaluation", {})
+
+    if isinstance(evaluation, str):
+        try:
+            evaluation = json.loads(evaluation)
+        except Exception as exc:
+            raise ValueError("`evaluation` is string but not valid JSON.") from exc
+
+    if not isinstance(evaluation, dict):
+        raise ValueError("`evaluation` must be a dict or JSON string.")
+
+    artifacts = evaluation.get("artifacts", {})
+    if not isinstance(artifacts, dict):
+        raise ValueError("`evaluation.artifacts` is missing or invalid.")
+
+    submission_file_path = artifacts.get("submission_file_path")
+    if not isinstance(submission_file_path, str) or not submission_file_path.strip():
+        raise ValueError("`evaluation.artifacts.submission_file_path` is missing or invalid.")
+
+    return submission_file_path.strip()
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Locate latest checkpoint and extract submission_file_path."
+    )
+    parser.add_argument(
+        "--checkpoints-root",
+        type=Path,
+        required=True,
+        help="Path to checkpoints root directory, e.g. output/database/checkpoints",
+    )
+    parser.add_argument(
+        "--field",
+        choices=("checkpoint_path", "best_solution_path", "submission_file_path", "json"),
+        default="json",
+        help="Which field to print.",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+
+    try:
+        checkpoint_path = find_latest_checkpoint_dir(args.checkpoints_root)
+        best_solution_path = checkpoint_path / "best_solution.json"
+        submission_file_path = extract_submission_path(best_solution_path)
+    except Exception as exc:
+        print(f"[ERROR] {exc}", file=sys.stderr)
+        return 1
+
+    payload = {
+        "checkpoint_path": str(checkpoint_path),
+        "best_solution_path": str(best_solution_path),
+        "submission_file_path": submission_file_path,
+    }
+
+    if args.field == "json":
+        print(json.dumps(payload, ensure_ascii=False))
+    else:
+        print(payload[args.field])
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())