[sled-agent] Integrate config-reconciler (#8064)

Make `sled-agent` use the new `sled-agent-config-reconciler`. Major visible changes:

* We go from three ledgers (disks, datasets, zones) to one (`omicron-sled-config.json`).
* The sled-agent `PUT /omicron-config` is now mostly asynchronous: it returns after ledgering the new config but before applying it.
* Inventory reports from sled-agent now include `ledgered_sled_config` (the current contents of the ledger), `reconciler_status` (status of the config reconciliation task, including whether it's running or idle), and `last_reconciliation` (status of the most recent completed reconciliation attempt, including the status of all configured disks, datasets, and zones).

Author: jgallagher

Cargo.lock

@@ -67,7 +67,6 @@ progenitor::generate_api!(
         OmicronPhysicalDiskConfig = omicron_common::disk::OmicronPhysicalDiskConfig,
         OmicronPhysicalDisksConfig = omicron_common::disk::OmicronPhysicalDisksConfig,
         OmicronSledConfig = nexus_sled_agent_shared::inventory::OmicronSledConfig,
-        OmicronSledConfigResult = nexus_sled_agent_shared::inventory::OmicronSledConfigResult,
         OmicronZoneConfig = nexus_sled_agent_shared::inventory::OmicronZoneConfig,
         OmicronZoneDataset = nexus_sled_agent_shared::inventory::OmicronZoneDataset,
         OmicronZoneImageSource = nexus_sled_agent_shared::inventory::OmicronZoneImageSource,

clients/sled-agent-client/src/lib.rs

@@ -7351,30 +7351,28 @@ fn inv_collection_print_sleds(collection: &Collection) {
                     "LAST RECONCILED CONFIG",
                     &last_reconciliation.last_reconciled_config,
                 );
-                let disk_errs = collect_config_reconciler_errors(
-                    &last_reconciliation.external_disks,
-                );
-                let dataset_errs = collect_config_reconciler_errors(
-                    &last_reconciliation.datasets,
-                );
-                let zone_errs = collect_config_reconciler_errors(
-                    &last_reconciliation.zones,
-                );
-                for (label, errs) in [
-                    ("disk", disk_errs),
-                    ("dataset", dataset_errs),
-                    ("zone", zone_errs),
-                ] {
-                    if errs.is_empty() {
-                        println!("    all {label}s reconciled successfully");
-                    } else {
-                        println!(
-                            "    {} {label} reconciliation errors:",
-                            errs.len()
-                        );
-                        for err in errs {
-                            println!("      {err}");
-                        }
+            }
+            let disk_errs = collect_config_reconciler_errors(
+                &last_reconciliation.external_disks,
+            );
+            let dataset_errs =
+                collect_config_reconciler_errors(&last_reconciliation.datasets);
+            let zone_errs =
+                collect_config_reconciler_errors(&last_reconciliation.zones);
+            for (label, errs) in [
+                ("disk", disk_errs),
+                ("dataset", dataset_errs),
+                ("zone", zone_errs),
+            ] {
+                if errs.is_empty() {
+                    println!("        all {label}s reconciled successfully");
+                } else {
+                    println!(
+                        "        {} {label} reconciliation errors:",
+                        errs.len()
+                    );
+                    for err in errs {
+                        println!("          {err}");
                     }
                 }
             }

dev-tools/omdb/src/bin/omdb/db.rs

@@ -34,14 +34,6 @@ enum SledAgentCommands {
     #[clap(subcommand)]
     Zones(ZoneCommands),
 
-    /// print information about zpools
-    #[clap(subcommand)]
-    Zpools(ZpoolCommands),
-
-    /// print information about datasets
-    #[clap(subcommand)]
-    Datasets(DatasetCommands),
-
     /// print information about the local bootstore node
     #[clap(subcommand)]
     Bootstore(BootstoreCommands),
@@ -97,12 +89,6 @@ impl SledAgentArgs {
             SledAgentCommands::Zones(ZoneCommands::List) => {
                 cmd_zones_list(&client).await
             }
-            SledAgentCommands::Zpools(ZpoolCommands::List) => {
-                cmd_zpools_list(&client).await
-            }
-            SledAgentCommands::Datasets(DatasetCommands::List) => {
-                cmd_datasets_list(&client).await
-            }
             SledAgentCommands::Bootstore(BootstoreCommands::Status) => {
                 cmd_bootstore_status(&client).await
             }
@@ -129,44 +115,6 @@ async fn cmd_zones_list(
     Ok(())
 }
 
-/// Runs `omdb sled-agent zpools list`
-async fn cmd_zpools_list(
-    client: &sled_agent_client::Client,
-) -> Result<(), anyhow::Error> {
-    let response = client.zpools_get().await.context("listing zpools")?;
-    let zpools = response.into_inner();
-
-    println!("zpools:");
-    if zpools.is_empty() {
-        println!("    <none>");
-    }
-    for zpool in &zpools {
-        println!("    {:?}", zpool);
-    }
-
-    Ok(())
-}
-
-/// Runs `omdb sled-agent datasets list`
-async fn cmd_datasets_list(
-    client: &sled_agent_client::Client,
-) -> Result<(), anyhow::Error> {
-    let response = client.datasets_get().await.context("listing datasets")?;
-    let response = response.into_inner();
-
-    println!("dataset configuration @ generation {}:", response.generation);
-    let datasets = response.datasets;
-
-    if datasets.is_empty() {
-        println!("    <none>");
-    }
-    for dataset in &datasets {
-        println!("    {:?}", dataset);
-    }
-
-    Ok(())
-}
-
 /// Runs `omdb sled-agent bootstore status`
 async fn cmd_bootstore_status(
     client: &sled_agent_client::Client,

dev-tools/omdb/src/bin/omdb/sled_agent.rs

@@ -107,7 +107,6 @@ async fn test_omdb_usage_errors() {
         &["nexus", "sleds"],
         &["sled-agent"],
         &["sled-agent", "zones"],
-        &["sled-agent", "zpools"],
         &["oximeter", "--help"],
         &["oxql", "--help"],
         // Mispelled argument

dev-tools/omdb/tests/test_all_output.rs

@@ -909,8 +909,6 @@ Usage: omdb sled-agent [OPTIONS] <COMMAND>
 
 Commands:
   zones      print information about zones
-  zpools     print information about zpools
-  datasets   print information about datasets
   bootstore  print information about the local bootstore node
   help       Print this message or the help of the given subcommand(s)
 
@@ -949,32 +947,6 @@ Connection Options:
   --sled-agent-url <SLED_AGENT_URL>  URL of the Sled internal API [env: OMDB_SLED_AGENT_URL=]
   --dns-server <DNS_SERVER>          [env: OMDB_DNS_SERVER=]
 
-Safety Options:
-  -w, --destructive  Allow potentially-destructive subcommands
-=============================================
-EXECUTING COMMAND: omdb ["sled-agent", "zpools"]
-termination: Exited(2)
----------------------------------------------
-stdout:
----------------------------------------------
-stderr:
-print information about zpools
-
-Usage: omdb sled-agent zpools [OPTIONS] <COMMAND>
-
-Commands:
-  list  Print list of all zpools managed by the sled agent
-  help  Print this message or the help of the given subcommand(s)
-
-Options:
-      --log-level <LOG_LEVEL>  log level filter [env: LOG_LEVEL=] [default: warn]
-      --color <COLOR>          Color output [default: auto] [possible values: auto, always, never]
-  -h, --help                   Print help
-
-Connection Options:
-  --sled-agent-url <SLED_AGENT_URL>  URL of the Sled internal API [env: OMDB_SLED_AGENT_URL=]
-  --dns-server <DNS_SERVER>          [env: OMDB_DNS_SERVER=]
-
 Safety Options:
   -w, --destructive  Allow potentially-destructive subcommands
 =============================================

dev-tools/omdb/tests/usage_errors.out

@@ -12,16 +12,14 @@ use chrono::{DateTime, Utc};
 use daft::Diffable;
 use id_map::IdMap;
 use id_map::IdMappable;
+use omicron_common::disk::{DatasetKind, DatasetName};
 use omicron_common::ledger::Ledgerable;
 use omicron_common::{
     api::{
         external::{ByteCount, Generation},
         internal::shared::{NetworkInterface, SourceNatConfig},
     },
-    disk::{
-        DatasetConfig, DatasetManagementStatus, DiskManagementStatus,
-        DiskVariant, OmicronPhysicalDiskConfig,
-    },
+    disk::{DatasetConfig, DiskVariant, OmicronPhysicalDiskConfig},
     update::ArtifactId,
     zpool_name::ZpoolName,
 };
@@ -132,6 +130,49 @@ pub struct ConfigReconcilerInventory {
     pub zones: BTreeMap<OmicronZoneUuid, ConfigReconcilerInventoryResult>,
 }
 
+impl ConfigReconcilerInventory {
+    /// Iterate over all running zones as reported by the last reconciliation
+    /// result.
+    ///
+    /// This includes zones that are both present in `last_reconciled_config`
+    /// and whose status in `zones` indicates "successfully running".
+    pub fn running_omicron_zones(
+        &self,
+    ) -> impl Iterator<Item = &OmicronZoneConfig> {
+        self.zones.iter().filter_map(|(zone_id, result)| match result {
+            ConfigReconcilerInventoryResult::Ok => {
+                self.last_reconciled_config.zones.get(zone_id)
+            }
+            ConfigReconcilerInventoryResult::Err { .. } => None,
+        })
+    }
+
+    /// Given a sled config, produce a reconciler result that sled-agent could
+    /// have emitted if reconciliation succeeded.
+    ///
+    /// This method should only be used by tests and dev tools; real code should
+    /// look at the actual `last_reconciliation` value from the parent
+    /// [`Inventory`].
+    pub fn debug_assume_success(config: OmicronSledConfig) -> Self {
+        let external_disks = config
+            .disks
+            .iter()
+            .map(|d| (d.id, ConfigReconcilerInventoryResult::Ok))
+            .collect();
+        let datasets = config
+            .datasets
+            .iter()
+            .map(|d| (d.id, ConfigReconcilerInventoryResult::Ok))
+            .collect();
+        let zones = config
+            .zones
+            .iter()
+            .map(|z| (z.id, ConfigReconcilerInventoryResult::Ok))
+            .collect();
+        Self { last_reconciled_config: config, external_disks, datasets, zones }
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Deserialize, JsonSchema, Serialize)]
 #[serde(tag = "result", rename_all = "snake_case")]
 pub enum ConfigReconcilerInventoryResult {
@@ -187,8 +228,6 @@ pub enum SledRole {
 }
 
 /// Describes the set of Reconfigurator-managed configuration elements of a sled
-// TODO this struct should have a generation number; at the moment, each of
-// the fields has a separete one internally.
 #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
 pub struct OmicronSledConfig {
     pub generation: Generation,
@@ -223,14 +262,6 @@ impl Ledgerable for OmicronSledConfig {
     }
 }
 
-/// Result of the currently-synchronous `omicron_config_put` endpoint.
-#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
-#[must_use = "this `DatasetManagementResult` may contain errors, which should be handled"]
-pub struct OmicronSledConfigResult {
-    pub disks: Vec<DiskManagementStatus>,
-    pub datasets: Vec<DatasetManagementStatus>,
-}
-
 /// Describes the set of Omicron-managed zones running on a sled
 #[derive(
     Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
@@ -297,6 +328,10 @@ impl OmicronZoneConfig {
             Some(self.id),
         )
     }
+
+    pub fn dataset_name(&self) -> Option<DatasetName> {
+        self.zone_type.dataset_name()
+    }
 }
 
 /// Describes a persistent ZFS dataset associated with an Omicron zone
@@ -583,6 +618,41 @@ impl OmicronZoneType {
             | OmicronZoneType::Oximeter { .. } => None,
         }
     }
+
+    /// If this kind of zone has an associated dataset, return the dataset's
+    /// name. Otherwise, return `None`.
+    pub fn dataset_name(&self) -> Option<DatasetName> {
+        let (dataset, dataset_kind) = match self {
+            OmicronZoneType::BoundaryNtp { .. }
+            | OmicronZoneType::InternalNtp { .. }
+            | OmicronZoneType::Nexus { .. }
+            | OmicronZoneType::Oximeter { .. }
+            | OmicronZoneType::CruciblePantry { .. } => None,
+            OmicronZoneType::Clickhouse { dataset, .. } => {
+                Some((dataset, DatasetKind::Clickhouse))
+            }
+            OmicronZoneType::ClickhouseKeeper { dataset, .. } => {
+                Some((dataset, DatasetKind::ClickhouseKeeper))
+            }
+            OmicronZoneType::ClickhouseServer { dataset, .. } => {
+                Some((dataset, DatasetKind::ClickhouseServer))
+            }
+            OmicronZoneType::CockroachDb { dataset, .. } => {
+                Some((dataset, DatasetKind::Cockroach))
+            }
+            OmicronZoneType::Crucible { dataset, .. } => {
+                Some((dataset, DatasetKind::Crucible))
+            }
+            OmicronZoneType::ExternalDns { dataset, .. } => {
+                Some((dataset, DatasetKind::ExternalDns))
+            }
+            OmicronZoneType::InternalDns { dataset, .. } => {
+                Some((dataset, DatasetKind::InternalDns))
+            }
+        }?;
+
+        Some(DatasetName::new(dataset.pool_name, dataset_kind))
+    }
 }
 
 /// Like [`OmicronZoneType`], but without any associated data.

nexus-sled-agent-shared/src/inventory.rs

@@ -16,7 +16,6 @@ use gateway_client::types::SpType;
 use gateway_types::rot::RotSlot;
 use nexus_sled_agent_shared::inventory::Baseboard;
 use nexus_sled_agent_shared::inventory::ConfigReconcilerInventory;
-use nexus_sled_agent_shared::inventory::ConfigReconcilerInventoryResult;
 use nexus_sled_agent_shared::inventory::ConfigReconcilerInventoryStatus;
 use nexus_sled_agent_shared::inventory::Inventory;
 use nexus_sled_agent_shared::inventory::InventoryDataset;
@@ -643,29 +642,9 @@ pub fn sled_agent(
     ledgered_sled_config: Option<OmicronSledConfig>,
 ) -> Inventory {
     // Assume the `ledgered_sled_config` was reconciled successfully.
-    let last_reconciliation = ledgered_sled_config.clone().map(|config| {
-        let external_disks = config
-            .disks
-            .iter()
-            .map(|d| (d.id, ConfigReconcilerInventoryResult::Ok))
-            .collect();
-        let datasets = config
-            .datasets
-            .iter()
-            .map(|d| (d.id, ConfigReconcilerInventoryResult::Ok))
-            .collect();
-        let zones = config
-            .zones
-            .iter()
-            .map(|z| (z.id, ConfigReconcilerInventoryResult::Ok))
-            .collect();
-        ConfigReconcilerInventory {
-            last_reconciled_config: config,
-            external_disks,
-            datasets,
-            zones,
-        }
-    });
+    let last_reconciliation = ledgered_sled_config
+        .clone()
+        .map(ConfigReconcilerInventory::debug_assume_success);
 
     let reconciler_status = if last_reconciliation.is_some() {
         ConfigReconcilerInventoryStatus::Idle {