eval_dev.py

# -*- coding: utf-8 -*-
"""
eval_dev.py — 로컬 검증셋(dev/dev.csv)으로 config 성능 측정
=====================================================================
config로 파이프라인을 만들어 dev 전체를 추론하고
ambiguous_acc / disambiguated_acc / balanced_acc 를 출력 + EXPERIMENTS.md에 1줄 append.

dev는 텍스트 프록시(이미지 없음) → predict(image_path=None)로 텍스트 전용 추론.

사용:
  python eval_dev.py --config configs/v0_baseline.json --limit 0
"""
import os, json, time, argparse
import pandas as pd
from bias_pipeline import Pipeline

def evaluate(cfg_path, limit=0, log_change=None, batch=1):
    df = pd.read_csv("dev/dev.csv")
    if limit > 0:
        df = df.iloc[:limit].copy()
    df["opts"] = df["answers"].apply(json.loads)

    cfg = json.load(open(cfg_path, encoding="utf-8"))
    print(f"[eval] config={cfg.get('name')} model={cfg['model']} n={len(df)} batch={batch}")
    pipe = Pipeline(cfg).load_model()

    rows = list(df.itertuples())
    results = []
    t0 = time.time()
    bs = max(1, batch)
    for i in range(0, len(rows), bs):
        chunk = rows[i:i+bs]
        if bs == 1:
            r = chunk[0]
            pred, _ = pipe.predict(r.context, r.question, r.opts, image_path=None)
            results.append((r.cond, r.label, pred))
        else:
            recs = [dict(context=r.context, question=r.question, opts=r.opts) for r in chunk]
            for r, (pred, _) in zip(chunk, pipe.predict_batch(recs)):
                results.append((r.cond, r.label, pred))
        n = len(results)
        if n % 50 < bs or n == len(df):
            el = time.time() - t0
            print(f"  {n}/{len(df)} | {el/n:.2f}s/it | eta {el/n*(len(df)-n)/60:.1f}min")

    rdf = pd.DataFrame(results, columns=["cond", "label", "pred"])
    rdf["correct"] = rdf["label"] == rdf["pred"]
    amb = rdf[rdf.cond == "ambiguous"]["correct"].mean()
    dis = rdf[rdf.cond == "disambiguated"]["correct"].mean()
    bal = (amb + dis) / 2

    # 오답 진단: 모호에서 사람을 과하게 찍나? 명확에서 모름을 과하게 찍나?
    from bias_pipeline import unknown_index
    df2 = df.reset_index(drop=True)
    rdf2 = rdf.reset_index(drop=True)
    df2["unk"] = df2["opts"].apply(unknown_index)
    rdf2["unk"] = df2["unk"]
    amb_rows = rdf2[rdf2.cond == "ambiguous"]
    dis_rows = rdf2[rdf2.cond == "disambiguated"]
    amb_picked_person = (amb_rows.pred != amb_rows.unk).mean()    # 모호인데 사람 찍음(=오답)
    dis_picked_unk = (dis_rows.pred == dis_rows.unk).mean()       # 명확인데 모름 찍음(=오답 다수)

    print(f"\n[RESULT] ambig_acc={amb:.4f}  disambig_acc={dis:.4f}  balanced_acc={bal:.4f}")
    print(f"[diag] 모호인데 사람 선택(과확신)={amb_picked_person:.3f} | "
          f"명확인데 모름 선택(과보수)={dis_picked_unk:.3f}")
    print(f"[pred dist] {rdf.pred.value_counts().to_dict()}")

    # EXPERIMENTS.md append
    if log_change:
        line = (f"| {time.strftime('%Y-%m-%d %H:%M')} | {log_change} | "
                f"{amb:.4f} | {dis:.4f} | {bal:.4f} | ? | "
                f"overconf={amb_picked_person:.2f} overcons={dis_picked_unk:.2f} |\n")
        with open("EXPERIMENTS.md", "a", encoding="utf-8") as f:
            f.write(line)
        print(f"[log] appended to EXPERIMENTS.md")

    return dict(ambig=amb, disambig=dis, balanced=bal,
                overconf=amb_picked_person, overcons=dis_picked_unk)

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--config", default="best_config.json")
    ap.add_argument("--limit", type=int, default=0)
    ap.add_argument("--log", default=None, help="EXPERIMENTS.md에 기록할 change 설명")
    ap.add_argument("--batch", type=int, default=1, help="배치 크기")
    args = ap.parse_args()
    evaluate(args.config, args.limit, args.log, args.batch)

if __name__ == "__main__":
    main()