feat: ✨ enhance dashboard with daily retrieval metrics and new visual…

…ization

- Replaced "wide" theme with "dashboard" for improved aesthetics.
- Introduced a script to calculate daily retrieval metrics using DuckDB.
- Updated dashboard to include sections for "Retrieval Service Class Conformance" and "Durability" with relevant visualizations.
- Bumped `lru-cache` dependency to v11.0.2 to ensure compatibility and security.

observablehq.config.js

@@ -19,8 +19,7 @@ export default {
   root: "src",
   theme: [
     "parchment",
-    "alt",
-    "wide", // 2024-10-30: this didn't have the expected impact
+    "dashboard"
   ],
   footer: false,
   sidebar: false,

src/data/daily_retrieval_metrics.csv.sh

@@ -0,0 +1,33 @@
+
+#!/usr/bin/env bash
+
+duckdb :memory: << EOF
+SET enable_progress_bar = false;
+COPY (
+    with retrieval_rates as (
+    select
+        date,
+        provider_id,
+        case
+        when total_retrieval_requests > 0
+        then successful_retrieval_requests * 1.0 / total_retrieval_requests
+        else 0
+        end as retrieval_rate
+    from read_parquet('https://data.filecoindataportal.xyz/filecoin_daily_storage_providers_metrics.parquet')
+    where date >= current_date() - interval '90 days'
+    )
+
+    select
+    date,
+    count(distinct provider_id) as meet_retrieval_sli,
+    count(distinct provider_id) * 1.0 / (
+        select count(distinct provider_id)
+        from retrieval_rates
+        where date = r.date
+    ) as meet_retrieval_sli_percent
+    from retrieval_rates r
+    where retrieval_rate > 0.9
+    group by date
+    order by date desc
+) TO STDOUT (FORMAT 'CSV');
+EOF

src/index.md

@@ -11,6 +11,7 @@ This dasbhoard is a scrappy work-in-progress to help spur discussion at [FDS-5 B
 
 ```js
 const daily_metrics = FileAttachment("data/daily_metrics.csv").csv({typed: true});
+const daily_retrieval_metrics = FileAttachment("data/daily_retrieval_metrics.csv").csv({typed: true});
 
 // Mutates the provided plotConfig by adding a caption if one doesn't exist
 function addPlotCaption(plotConfig, href) {
@@ -109,6 +110,60 @@ This is an aggregate view looking at all Storage Providers on the network.
   </div>
 </div>
 
+### Retrieval Service Class Conformance
+
+<div class="grid grid-cols-2">
+  <div class="card">
+
+  ```js
+  resize((width) => Plot.plot(addPlotCaption({
+    title: "Providers Meeting Retrieval SLI",
+    subtitle: "Number of providers with >90% successful retrievals",
+    x: { label: "Date" },
+    y: { grid: true, label: "Providers", zero: true },
+    width,
+    marks: [
+      Plot.ruleY([0]),
+      Plot.lineY(daily_retrieval_metrics, {
+        x: "date",
+        y: "meet_retrieval_sli",
+        tip: true
+      })
+    ]
+  })))
+  ```
+
+  </div>
+
+  <div class="card">
+
+  ```js
+  resize((width) => Plot.plot(addPlotCaption({
+    title: "Providers Meeting Retrieval SLI (%)",
+    subtitle: "Percentage of providers with >90% successful retrievals",
+    x: { label: "Date" },
+    y: {
+      grid: true,
+      label: "Percentage",
+      domain: [0, 100]
+    },
+    width,
+    marks: [
+      Plot.ruleY([0]),
+      Plot.lineY(daily_retrieval_metrics, {
+        x: "date",
+        y: d => d.meet_retrieval_sli_percent * 100,
+        tip: true
+      })
+    ]
+  })))
+  ```
+
+  </div>
+</div>
+
+
+
 ### Service Class Conformance
 
 <div class="grid grid-cols-2">

src/provider/[provider].md

@@ -10,7 +10,6 @@
 const provider_metrics = FileAttachment(`../data/${observable.params.provider}_daily_metrics.csv`).csv({typed: true});
 ```
 
-
 ## Retrievals
 
 <div class="grid grid-cols-2">
@@ -72,6 +71,37 @@ const provider_metrics = FileAttachment(`../data/${observable.params.provider}_d
 </div>
 </div>
 
+## Durability
+
+<div class="card">
+
+```js
+resize((width) => Plot.plot({
+  title: "Durability",
+  subtitle: "Percentage of sectors that have faulted on a given day",
+  x: { label: "Date" },
+  y: {
+    grid: true,
+    label: "Fault Rate (%)",
+    domain: [0, 100]
+  },
+  width,
+  marks: [
+    Plot.ruleY([0]),
+    Plot.text(
+      ["To be implemented"],
+      {
+        x: 0,
+        y: 40,
+        fontSize: 48,
+        fill: "var(--theme-foreground-faint)"
+      }
+    )
+  ]
+}))
+```
+
+</div>
 
 ## Other Metrics