|
| 1 | +{ |
| 2 | + "last_updated": "2026-04-03", |
| 3 | + "paper_url": "https://arxiv.org/abs/2604.00986", |
| 4 | + "code_url": "https://github.com/FreedomIntelligence/MyPhoneBench", |
| 5 | + "blog_url": "https://www.notion.so/Do-Phone-Use-Agents-Respect-Your-Privacy-The-Missing-Privacy-Gate-Before-Phone-Agents-Can-Truly-S-337f1aa81ec6808daf3ad88b6baecd67", |
| 6 | + "trajectory_url": "https://modelscope.cn/datasets/tangzhy/MyPhoneBench-Trajectories", |
| 7 | + "ranking_method": "Normalized Average Score: min-max normalize each metric to [0,100], then average across all 4 metrics. Higher is better.", |
| 8 | + "metrics": [ |
| 9 | + {"key": "task_sr", "name": "Task SR (%)", "description": "Task success rate via deterministic SQL verification", "higher_is_better": true}, |
| 10 | + {"key": "privacy", "name": "Privacy (%)", "description": "Equal-weight average of Over-Permissioning, Trap Resistance, Form Minimization", "higher_is_better": true}, |
| 11 | + {"key": "pqsr", "name": "PQSR@0.7 (%)", "description": "Privacy-Qualified Success Rate: task completed AND privacy score >= 0.7", "higher_is_better": true}, |
| 12 | + {"key": "later_session", "name": "Later-Session (%)", "description": "Correct use of saved preferences in paired cross-session tasks", "higher_is_better": true}, |
| 13 | + {"key": "overall", "name": "Overall", "description": "Average of min-max normalized scores across all 4 metrics (higher is better)", "higher_is_better": true} |
| 14 | + ], |
| 15 | + "models": [ |
| 16 | + { |
| 17 | + "name": "Claude Opus 4.6", |
| 18 | + "org": "Anthropic", |
| 19 | + "task_sr": 82.8, |
| 20 | + "privacy": 68.4, |
| 21 | + "pqsr": 47.2, |
| 22 | + "later_session": 72.0 |
| 23 | + }, |
| 24 | + { |
| 25 | + "name": "Qwen 3.5 Plus", |
| 26 | + "org": "Alibaba", |
| 27 | + "task_sr": 76.0, |
| 28 | + "privacy": 73.8, |
| 29 | + "pqsr": 47.6, |
| 30 | + "later_session": 48.0 |
| 31 | + }, |
| 32 | + { |
| 33 | + "name": "Kimi K2.5", |
| 34 | + "org": "Moonshot", |
| 35 | + "task_sr": 65.2, |
| 36 | + "privacy": 77.3, |
| 37 | + "pqsr": 45.2, |
| 38 | + "later_session": 58.0 |
| 39 | + }, |
| 40 | + { |
| 41 | + "name": "Doubao Seed 1.8", |
| 42 | + "org": "ByteDance", |
| 43 | + "task_sr": 57.2, |
| 44 | + "privacy": 71.0, |
| 45 | + "pqsr": 31.2, |
| 46 | + "later_session": 42.0 |
| 47 | + }, |
| 48 | + { |
| 49 | + "name": "Gemini 3 Pro", |
| 50 | + "org": "Google", |
| 51 | + "task_sr": 50.4, |
| 52 | + "privacy": 60.5, |
| 53 | + "pqsr": 22.0, |
| 54 | + "later_session": 20.0 |
| 55 | + } |
| 56 | + ] |
| 57 | +} |
0 commit comments