plots/ai-benchmark-progress/output/benchmark_progress_interactive.html at 1642c1d8dd4027e5ff108f75161d097f96fa38c4 · mschwar/plots · GitHub

1
2
3
4
5
6
7
<html>
<head><meta charset="utf-8" /></head>
<body>
    <div>                        <script>window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
        <script charset="utf-8" src="https://cdn.plot.ly/plotly-3.5.0.min.js" integrity="sha256-fHbNLP+GlIXN+efbQec78UkemUz3NJp7UmfGxC1tNxs=" crossorigin="anonymous"></script>                <div id="0c82f28b-b3a9-48af-8ab2-63a7092e4dd7" class="plotly-graph-div" style="height:600px; width:100%;"></div>            <script>                window.PLOTLYENV=window.PLOTLYENV || {};                                if (document.getElementById("0c82f28b-b3a9-48af-8ab2-63a7092e4dd7")) {                    Plotly.newPlot(                        "0c82f28b-b3a9-48af-8ab2-63a7092e4dd7",                        [{"hovertemplate":"%{text}\u003cbr\u003eYear: %{x}\u003cbr\u003eScore: %{y:.1f}%\u003cextra\u003e\u003c\u002fextra\u003e","line":{"color":"#60a5fa","width":3},"marker":{"size":10},"mode":"lines+markers","name":"MMLU \u2014 Score","showlegend":true,"text":["GPT-2 (OpenAI)","GPT-3 (OpenAI)","GPT-3.5 (OpenAI)","Chinchilla (DeepMind)","GPT-4 (OpenAI)","Gemini 1.5 Pro (Google)","Gemini 2.0 Pro (Google)"],"x":{"dtype":"i2","bdata":"4wfkB+UH5gfnB+gH6Qc="},"y":{"dtype":"f8","bdata":"AAAAAACATEAAAAAAAABRQAAAAAAAQFJAAAAAAADAUkCamZmZmZlVQDMzMzMz01dAZmZmZmaGWEA="},"type":"scatter","xaxis":"x","yaxis":"y"},{"line":{"color":"#ef4444","dash":"dash","width":2},"mode":"lines","name":"Human Baseline","showlegend":true,"x":[2019,2025],"y":[89.8,89.8],"type":"scatter","xaxis":"x","yaxis":"y"},{"hovertemplate":"%{text}\u003cbr\u003eYear: %{x}\u003cbr\u003eScore: %{y:.1f}%\u003cextra\u003e\u003c\u002fextra\u003e","line":{"color":"#34d399","width":3},"marker":{"size":10},"mode":"lines+markers","name":"HumanEval \u2014 Score","showlegend":false,"text":["Codex (OpenAI)","PaLM-Coder (Google)","GPT-4 (OpenAI)","Gemini 1.5 Pro (Google)","o3 (OpenAI)"],"x":{"dtype":"i2","bdata":"5QfmB+cH6AfpBw=="},"y":{"dtype":"f8","bdata":"zczMzMzMPEAAAAAAAMBHQAAAAAAAwFBAAAAAAAAAV0AAAAAAACBYQA=="},"type":"scatter","xaxis":"x2","yaxis":"y2"},{"line":{"color":"#ef4444","dash":"dash","width":2},"mode":"lines","showlegend":false,"x":[2021,2025],"y":[72.0,72.0],"type":"scatter","xaxis":"x2","yaxis":"y2"},{"hovertemplate":"%{text}\u003cbr\u003eYear: %{x}\u003cbr\u003eScore: %{y:.1f}%\u003cextra\u003e\u003c\u002fextra\u003e","line":{"color":"#9ca3af","width":3},"marker":{"size":10},"mode":"lines+markers","name":"SWE-bench \u2014 Score","showlegend":false,"text":["Claude 3.5 Sonnet (Anthropic)","GPT-4o (OpenAI)","Claude 3.7 Sonnet (Anthropic)","o3 (OpenAI)"],"x":{"dtype":"i2","bdata":"6AfoB+kH6Qc="},"y":{"dtype":"f8","bdata":"AAAAAACARUAAAAAAAABMQM3MzMzM7FFAZmZmZmbmVUA="},"type":"scatter","xaxis":"x3","yaxis":"y3"},{"line":{"color":"#ef4444","dash":"dash","width":2},"mode":"lines","showlegend":false,"x":[2024,2025],"y":[100.0,100.0],"type":"scatter","xaxis":"x3","yaxis":"y3"},{"hovertemplate":"%{text}\u003cbr\u003eYear: %{x}\u003cbr\u003eScore: %{y:.1f}%\u003cextra\u003e\u003c\u002fextra\u003e","line":{"color":"#a78bfa","width":3},"marker":{"size":10},"mode":"lines+markers","name":"ARC-AGI \u2014 Score","showlegend":false,"text":["GPT-3 (OpenAI)","GPT-3.5 (OpenAI)","GPT-4 (OpenAI)","GPT-4 Turbo (OpenAI)","o1 (OpenAI)","o3-mini (OpenAI)","o3 Pro (OpenAI)"],"x":{"dtype":"i2","bdata":"5AflB+YH5wfoB+kH6gc="},"y":{"dtype":"f8","bdata":"AAAAAACANEAAAAAAAAA5QAAAAAAAAD5AAAAAAACASkAAAAAAAABSQAAAAAAAQFVAAAAAAADAVkA="},"type":"scatter","xaxis":"x4","yaxis":"y4"},{"line":{"color":"#ef4444","dash":"dash","width":2},"mode":"lines","showlegend":false,"x":[2020,2026],"y":[85.0,85.0],"type":"scatter","xaxis":"x4","yaxis":"y4"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermap":[{"type":"scattermap","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,0.2125]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"range":[0,105]},"xaxis2":{"anchor":"y2","domain":[0.2625,0.475]},"yaxis2":{"anchor":"x2","domain":[0.0,1.0],"range":[0,105]},"xaxis3":{"anchor":"y3","domain":[0.525,0.7375]},"yaxis3":{"anchor":"x3","domain":[0.0,1.0],"range":[0,105]},"xaxis4":{"anchor":"y4","domain":[0.7875,1.0]},"yaxis4":{"anchor":"x4","domain":[0.0,1.0],"range":[0,105]},"annotations":[{"font":{"size":16},"showarrow":false,"text":"MMLU","x":0.10625,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"HumanEval","x":0.36875,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"SWE-bench","x":0.6312500000000001,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"ARC-AGI","x":0.89375,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"}],"title":{"font":{"size":20,"color":"#e8eaf6"},"text":"AI Benchmark Progress: Crossing Human Baselines","x":0.5},"font":{"color":"#d1d5db","family":"SF Pro Display, sans-serif"},"legend":{"font":{"color":"#d1d5db"},"bgcolor":"#1c1f2e","bordercolor":"#2d3148"},"margin":{"t":80,"b":60},"paper_bgcolor":"#0f1117","plot_bgcolor":"#0f1117","height":600},                        {"responsive": true}                    )                };            </script>        </div>
</body>
</html>