Add energy parity and per-element errors for sevennet-l3i5

- skip existing energy parity figs in single_model_parity_energy.py
- Add energy-parity Svelte figs for sevennet-0 and sevennet-l3i5
- Add per-element errors JSON for sevennet-l3i5
- inline CaptionedMetricsTable into landing page and delete the components

readme.md

@@ -1,4 +1,4 @@
-<h1 align="center" style="line-height: 0; margin-block: 0 1em;">
+<h1 align="center" style="line-height: 0; margin-block: -1em 1em;">
   <img src="https://github.com/janosh/matbench-discovery/raw/main/site/static/favicon.svg" alt="Logo" width="60px"><br>
   Matbench Discovery
 </h1>
@@ -14,9 +14,8 @@
 </h4>
 
 <slot name="metrics-table" />
-<slot name="table-controls" />
 
-Matbench Discovery is an [interactive leaderboard](https://janosh.github.io/matbench-discovery/models) and associated [PyPI package](https://pypi.org/project/matbench-discovery) which together make it easy to rank ML energy models on a task designed to simulate high-throughput discovery of new stable inorganic crystals.
+Matbench Discovery is an [interactive leaderboard](https://janosh.github.io/matbench-discovery/models) and associated [PyPI package](https://pypi.org/project/matbench-discovery) which rank ML models on a task designed to simulate high-throughput discovery of new stable inorganic crystals as well as predicting some of their properties.
 
 We've tested <slot name="model-count" /> models covering multiple methodologies including graph neural network (GNN) interatomic potentials, GNN one-shot predictors, iterative Bayesian optimizers and random forests with shallow-learning structure fingerprints.
 

scripts/model_figs/single_model_parity_energy.py

@@ -12,6 +12,7 @@
 from pymatviz.enums import Key
 
 from matbench_discovery import SITE_FIGS
+from matbench_discovery.data import Model
 from matbench_discovery.enums import MbdKey, TestSubset
 from matbench_discovery.preds.discovery import (
     df_each_pred,
@@ -26,6 +27,7 @@
 # toggle between formation energy and energy above convex hull
 EnergyType = Literal["e-form", "each"]
 use_e_form, use_each = get_args(EnergyType)
+update_existing: bool = False
 
 test_subset = globals().get("test_subset", TestSubset.uniq_protos)
 if test_subset == TestSubset.uniq_protos:
@@ -37,7 +39,13 @@
 parity_scatters_dir = f"{SITE_FIGS}/energy-parity"
 os.makedirs(parity_scatters_dir, exist_ok=True)
 
-for model, which_energy in itertools.product(df_metrics, (use_e_form, use_each)):
+for model_name, which_energy in itertools.product(df_metrics, (use_e_form, use_each)):
+    model_key = Model.from_label(model_name).key
+    img_name = f"{which_energy}-parity-{model_name.lower().replace(' ', '-')}"
+    img_path = f"{parity_scatters_dir}/{img_name}.svelte"
+    if os.path.isfile(img_path) and not update_existing:
+        continue
+
     if which_energy == use_each:
         df_in = df_each_pred.copy()
         df_in[MbdKey.each_true] = df_preds[MbdKey.each_true]
@@ -50,8 +58,8 @@
     else:
         raise ValueError(f"Unexpected {which_energy=}")
 
-    e_pred_col = f"{model} {e_true_col.label.replace('DFT ', '')}"
-    df_in = df_in.rename(columns={model: e_pred_col})
+    e_pred_col = f"{model_name} {e_true_col.label.replace('DFT ', '')}"
+    df_in = df_in.rename(columns={model_name: e_pred_col})
 
     fig = pmv.density_scatter_plotly(
         df=df_in.reset_index(drop=True),
@@ -63,11 +71,12 @@
         color_continuous_scale="agsunset",
     )
 
-    fig.layout.title.update(text=f"{model} {which_energy}", x=0.5)
+    # reduce colorbar size
+    fig.data[0].marker.colorbar.update(thickness=0.02)
+
+    fig.layout.title.update(text=f"{model_name} {which_energy}", x=0.5)
     fig.layout.margin.update(l=0, r=0, t=50, b=0)
 
     pmv.powerups.add_identity_line(fig)
     fig.show()
-
-    img_name = f"{which_energy}-parity-{model.lower().replace(' ', '-')}"
-    pmv.save_fig(fig, f"{parity_scatters_dir}/{img_name}.svelte")
+    pmv.save_fig(fig, img_path)

site/package.json

@@ -49,7 +49,7 @@
     "svelte-multiselect": "11.0.0-rc.1",
     "svelte-preprocess": "^6.0.3",
     "svelte-toc": "^0.5.9",
-    "svelte-zoo": "^0.4.14",
+    "svelte-zoo": "^0.4.15",
     "svelte2tsx": "^0.7.30",
     "tslib": "^2.8.1",
     "typescript": "5.7.2",

site/src/lib/HeatmapTable.svelte

@@ -7,6 +7,7 @@
   import { titles_as_tooltips } from 'svelte-zoo/actions'
   import { flip } from 'svelte/animate'
   import { writable } from 'svelte/store'
+  import type { HeatmapColumn } from './types'
 
   type TableData = Record<string, string | number | undefined>[]
 
@@ -117,7 +118,12 @@
       <!-- Second level headers -->
       <tr>
         {#each visible_columns as col, col_idx}
-          <th title={col.tooltip} on:click={() => sort_rows(col.label)} style={col.style}>
+          <th
+            title={col.tooltip}
+            on:click={() => sort_rows(col.label)}
+            style={col.style}
+            class:sticky-col={col.sticky}
+          >
             {@html col.label}
             {@html sort_indicator(col)}
             {#if col_idx == 0 && sort_hint}

site/src/lib/index.ts

@@ -8,7 +8,6 @@ import type { ModelMetadata } from './model-schema'
 import type { ModelData } from './types'
 
 export { default as TRAINING_SETS } from '$data/training-sets.yml'
-export { default as CaptionedMetricsTable } from './CaptionedMetricsTable.svelte'
 export { default as DiscoveryMetricsTable } from './DiscoveryMetricsTable.svelte'
 export { default as Footer } from './Footer.svelte'
 export { default as GeoOptMetricsTable } from './GeoOptMetricsTable.svelte'

site/src/routes/+page.svelte

@@ -1,11 +1,15 @@
 <script lang="ts">
   import type { ModelData } from '$lib'
-  import { CaptionedMetricsTable, model_is_compliant, MODEL_METADATA } from '$lib'
+  import { DiscoveryMetricsTable, model_is_compliant, MODEL_METADATA } from '$lib'
   import Readme from '$root/readme.md'
   import KappaNote from '$site/src/routes/kappa-note.md'
   import Icon from '@iconify/svelte'
+  import { pretty_num } from 'elementari'
   import { Toggle, Tooltip } from 'svelte-zoo'
 
+  let n_wbm_stable_uniq_protos = 32_942
+  let n_wbm_uniq_protos = 215_488
+
   let show_non_compliant: boolean = false
   let show_energy_only: boolean = false
 
@@ -54,54 +58,82 @@
 />
 
 <Readme>
-  <CaptionedMetricsTable
-    {show_non_compliant}
-    {show_energy_only}
-    {hide_cols}
-    style="margin-top: 4em;"
-    slot="metrics-table"
-  />
-  <div slot="table-controls">
-    <Toggle bind:checked={show_non_compliant} style="gap: 3pt;">
-      Show non-compliant models <Tooltip max_width="20em">
-        <span slot="tip">
-          Models can be non-compliant for multiple reasons<br />
-          - closed source (model implementation and/or train/test code)<br />
-          - closed weights<br />
-          - trained on more than the permissible training set (<a
-            href="https://docs.materialsproject.org/changes/database-versions#v2022.10.28"
-            >MP v2022.10.28 release</a
-          >)<br />
-          We still show these models behind a toggle as we expect them<br /> to nonetheless
-          provide helpful signals for developing future models.
-        </span>
-        <Icon icon="octicon:info-16" inline style="padding: 0 3pt;" />
-      </Tooltip>&ensp;</Toggle
-    >
-    <Toggle bind:checked={show_energy_only} style="gap: 3pt;">
-      Show energy-only models <Tooltip max_width="20em">
-        <span slot="tip">
-          Models that only predict energy (E) perform worse<br /> and can't be evaluated
-          on force-modeling tasks such as κ<sub>SRME</sub>
-        </span>
-        <Icon icon="octicon:info-16" inline style="padding: 0 3pt;" />
-      </Tooltip>&ensp;</Toggle
-    >
+  <figure style="margin-top: 4em;" slot="metrics-table">
+    <DiscoveryMetricsTable
+      {show_non_compliant}
+      {hide_cols}
+      {show_energy_only}
+      style="width: 100%;"
+    />
+    <div class="downloads">
+      Download table as
+      {#each [`PDF`, `SVG`] as file_ext}
+        {@const suffix = show_non_compliant ? `` : `-only-compliant`}
+        <a
+          href="/figs/metrics-table-uniq-protos{suffix}.{file_ext.toLowerCase()}"
+          download
+        >
+          {file_ext}
+        </a>
+      {/each}
+    </div>
+    <div class="table-controls">
+      <Toggle bind:checked={show_non_compliant} style="gap: 3pt;">
+        Show non-compliant models <Tooltip max_width="20em">
+          <span slot="tip">
+            Models can be non-compliant for multiple reasons<br />
+            - closed source (model implementation and/or train/test code)<br />
+            - closed weights<br />
+            - trained on more than the permissible training set (<a
+              href="https://docs.materialsproject.org/changes/database-versions#v2022.10.28"
+              >MP v2022.10.28 release</a
+            >)<br />
+            We still show these models behind a toggle as we expect them<br /> to nonetheless
+            provide helpful signals for developing future models.
+          </span>
+          <Icon icon="octicon:info-16" inline style="padding: 0 3pt;" />
+        </Tooltip>&ensp;</Toggle
+      >
+      <Toggle bind:checked={show_energy_only} style="gap: 3pt;">
+        Show energy-only models <Tooltip max_width="20em">
+          <span slot="tip">
+            Models that only predict energy (E) perform worse<br /> and can't be evaluated
+            on force-modeling tasks such as κ<sub>SRME</sub>
+          </span>
+          <Icon icon="octicon:info-16" inline style="padding: 0 3pt;" />
+        </Tooltip>&ensp;</Toggle
+      >
 
-    <details class="column-toggles" bind:open={column_panel_open}>
-      <summary>
-        Columns <Icon icon="octicon:columns-16" inline />
-      </summary>
-      <div class="column-menu">
-        {#each Object.keys(visible_cols) as col}
-          <label>
-            <input type="checkbox" bind:checked={visible_cols[col]} />
-            {@html col}
-          </label>
-        {/each}
-      </div>
-    </details>
-  </div>
+      <details class="column-toggles" bind:open={column_panel_open}>
+        <summary>
+          Columns <Icon icon="octicon:columns-16" inline />
+        </summary>
+        <div class="column-menu">
+          {#each Object.keys(visible_cols) as col}
+            <label>
+              <input type="checkbox" bind:checked={visible_cols[col]} />
+              {@html col}
+            </label>
+          {/each}
+        </div>
+      </details>
+    </div>
+    <figcaption>
+      Training size is the number of materials used to train the model. For models trained
+      on DFT relaxations, we show the number of distinct frames in parentheses. In cases
+      where only the number of frames is known, we report the number of frames as the
+      training set size. <code>(N=x)</code> in the Model Params column shows the number of
+      estimators if an ensemble was used. DAF = Discovery Acceleration Factor measures how
+      many more stable materials a model finds compared to random selection from the test
+      set. The unique structure prototypes in the WBM test set have a
+      <code>{pretty_num(n_wbm_stable_uniq_protos / n_wbm_uniq_protos, `.1%`)}</code> rate
+      of stable crystals, meaning the max possible DAF is
+      <code
+        >({pretty_num(n_wbm_stable_uniq_protos)} / {pretty_num(n_wbm_uniq_protos)})^−1 ≈
+        {pretty_num(n_wbm_uniq_protos / n_wbm_stable_uniq_protos)}</code
+      >.
+    </figcaption>
+  </figure>
 
   <span slot="model-count">
     {MODEL_METADATA.filter((md) => show_non_compliant || model_is_compliant(md)).length}
@@ -121,13 +153,33 @@
 <KappaNote />
 
 <style>
-  div[slot='table-controls'] {
+  figure {
+    margin: 0;
+    display: grid;
+    gap: 1ex;
+  }
+  figcaption {
+    font-size: 0.9em;
+    padding: 2pt 6pt;
+    background-color: rgba(255, 255, 255, 0.06);
+  }
+  div.downloads {
+    display: flex;
+    gap: 1ex;
+    justify-content: center;
+    margin-block: 1ex;
+  }
+  div.downloads a {
+    background-color: rgba(255, 255, 255, 0.1);
+    padding: 0 6pt;
+    border-radius: 4pt;
+  }
+  div.table-controls {
     display: flex;
     flex-wrap: wrap;
-    gap: 1em;
+    gap: 5pt;
     place-content: center;
-    margin: 2em auto;
-    max-width: 500px;
+    margin: 3pt auto;
   }
   .column-toggles {
     position: relative;
@@ -168,7 +220,7 @@
   .column-menu label:hover {
     background: rgba(255, 255, 255, 0.1);
   }
-  .column-menu :global(:is(sub, sup)) {
+  :is(figure[slot='metrics-table'], .column-menu) :global(:is(sub, sup)) {
     transform: translate(-3pt, 6pt);
     font-size: 0.7em;
   }