CallShield/docs/evaluation_results_20260301.json at master · melbinkm/CallShield · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
{
  "run_date": "2026-03-01",
  "backend_url": "https://callshield.onrender.com",
  "model": "mistral-large-latest",
  "total_scenarios": 25,
  "binary_accuracy": "25/25",
  "binary_accuracy_pct": 1.0,
  "precision": 1.0,
  "recall": 1.0,
  "specificity": 1.0,
  "f1": 1.0,
  "exact_match": "22/25",
  "hard_cases_correct": "3/3",
  "confusion_matrix": {
    "TP": 15,
    "FN": 0,
    "FP": 0,
    "TN": 10
  },
  "results": [
    {"id": "S01", "category": "IRS Arrest Threat",          "expected": "SCAM",        "score": 0.98, "verdict": "SCAM",        "binary_match": true, "exact_match": true},
    {"id": "S02", "category": "Tech Support Virus Alert",   "expected": "SCAM",        "score": 0.95, "verdict": "SCAM",        "binary_match": true, "exact_match": true},
    {"id": "S03", "category": "Medicare Robocall",          "expected": "LIKELY_SCAM", "score": 0.80, "verdict": "LIKELY_SCAM", "binary_match": true, "exact_match": true},
    {"id": "S04", "category": "Auto Warranty Robocall",     "expected": "LIKELY_SCAM", "score": 0.80, "verdict": "LIKELY_SCAM", "binary_match": true, "exact_match": true},
    {"id": "S05", "category": "Grandparent Scam",           "expected": "SCAM",        "score": 0.90, "verdict": "SCAM",        "binary_match": true, "exact_match": true},
    {"id": "S06", "category": "Romance Scam",               "expected": "LIKELY_SCAM", "score": 0.85, "verdict": "LIKELY_SCAM", "binary_match": true, "exact_match": true},
    {"id": "S07", "category": "Fake Bank Fraud Dept",       "expected": "SCAM",        "score": 0.85, "verdict": "SCAM",        "binary_match": true, "exact_match": true},
    {"id": "S08", "category": "Lottery Winner",             "expected": "SCAM",        "score": 0.95, "verdict": "SCAM",        "binary_match": true, "exact_match": true},
    {"id": "S09", "category": "Debt Threats Arrest",        "expected": "LIKELY_SCAM", "score": 0.90, "verdict": "LIKELY_SCAM", "binary_match": true, "exact_match": true},
    {"id": "S10", "category": "Crypto Guaranteed Returns",  "expected": "LIKELY_SCAM", "score": 0.90, "verdict": "LIKELY_SCAM", "binary_match": true, "exact_match": true},
    {"id": "L01", "category": "Friend Call",                "expected": "SAFE",        "score": 0.00, "verdict": "SAFE",        "binary_match": true, "exact_match": true},
    {"id": "L02", "category": "Meeting Scheduling",         "expected": "SAFE",        "score": 0.00, "verdict": "SAFE",        "binary_match": true, "exact_match": true},
    {"id": "L03", "category": "Doctor Reminder IVR",        "expected": "SAFE",        "score": 0.10, "verdict": "SAFE",        "binary_match": true, "exact_match": true},
    {"id": "L04", "category": "BBQ Invitation",             "expected": "SAFE",        "score": 0.00, "verdict": "SAFE",        "binary_match": true, "exact_match": true},
    {"id": "L05", "category": "Customer Service Callback",  "expected": "SAFE",        "score": 0.10, "verdict": "SAFE",        "binary_match": true, "exact_match": true},
    {"id": "L06", "category": "Angry Customer Complaint",   "expected": "SAFE",        "score": 0.10, "verdict": "SAFE",        "binary_match": true, "exact_match": true},
    {"id": "L07", "category": "Parent Dinner Plans",        "expected": "SAFE",        "score": 0.00, "verdict": "SAFE",        "binary_match": true, "exact_match": true},
    {"id": "L08", "category": "Job Interview Scheduling",   "expected": "SAFE",        "score": 0.05, "verdict": "SAFE",        "binary_match": true, "exact_match": true},
    {"id": "L09", "category": "Legit Bank Fraud Alert",     "expected": "SAFE",        "score": 0.15, "verdict": "SAFE",        "binary_match": true, "exact_match": true},
    {"id": "L10", "category": "Friend Voicemail",           "expected": "SAFE",        "score": 0.00, "verdict": "SAFE",        "binary_match": true, "exact_match": true},
    {"id": "A01", "category": "Polite IRS Gift Card",       "expected": "SCAM",        "score": 0.95, "verdict": "SCAM",        "binary_match": true, "exact_match": true},
    {"id": "A02", "category": "Hedged Crypto Pitch",        "expected": "LIKELY_SCAM", "score": 0.80, "verdict": "LIKELY_SCAM", "binary_match": true, "exact_match": true},
    {"id": "A03", "category": "Certified Tech Support",     "expected": "SCAM",        "score": 0.90, "verdict": "SCAM",        "binary_match": true, "exact_match": true},
    {"id": "A04", "category": "FDIC Bank Examiner",         "expected": "SCAM",        "score": 0.92, "verdict": "SCAM",        "binary_match": true, "exact_match": true},
    {"id": "A05", "category": "Romance to Investment",      "expected": "LIKELY_SCAM", "score": 0.85, "verdict": "LIKELY_SCAM", "binary_match": true, "exact_match": true}
  ],
  "notes": "A01-A05 are adversarial scenarios using evasive framing (polite tone, hedged language, false credentials) while retaining core scam signals. All 5 correctly classified at 25/25 binary accuracy.",
  "reproduce": "python scripts/run_evaluation.py --url https://callshield.onrender.com"
}