From 8a3d538973d5344f1f662f5bd9d185ad2a0eef00 Mon Sep 17 00:00:00 2001 From: James MacMahon Date: Tue, 6 May 2025 15:52:47 +0000 Subject: [PATCH 1/2] Allow configuring control_plane_storage_buffer Instead of a hard-coded const, allow configuring control plane storage buffer as a tunable. Fixes #7979 --- .github/buildomat/jobs/deploy.sh | 12 ------------ nexus-config/src/nexus_config.rs | 14 +++++++++++++- nexus/examples/config-second.toml | 3 +++ nexus/examples/config.toml | 3 +++ nexus/src/app/background/init.rs | 6 ++++++ .../tasks/physical_disk_adoption.rs | 7 +++++-- nexus/src/app/mod.rs | 19 +++++++++++-------- nexus/src/app/rack.rs | 3 +-- nexus/tests/config.test.toml | 3 +++ smf/nexus/multi-sled/config-partial.toml | 7 +++++++ smf/nexus/single-sled/config-partial.toml | 6 ++++++ 11 files changed, 58 insertions(+), 25 deletions(-) diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index 1fa0caf38e0..2e90ef0c675 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -429,18 +429,6 @@ do ACTUAL_ZPOOL_COUNT=$(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i | wc -l) done -# The bootstrap command creates a disk, so before that: adjust the control plane -# storage buffer to 0 as the virtual hardware only creates 20G pools - -pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list - -for ZPOOL in $(pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list -i); -do - pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb -w db zpool set-storage-buffer "${ZPOOL}" 0 -done - -pfexec zlogin oxz_switch /opt/oxide/omdb/bin/omdb db zpool list - export RUST_BACKTRACE=1 export E2E_TLS_CERT IPPOOL_START IPPOOL_END eval "$(./target/debug/bootstrap)" diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 1364a52fbad..e1a11d2275b 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -275,6 +275,7 @@ pub struct MgdConfig { struct UnvalidatedTunables { max_vpc_ipv4_subnet_prefix: u8, load_timeout: Option, + control_plane_storage_buffer_gb: u32, } /// Configuration for HTTP clients to external services. @@ -303,6 +304,11 @@ pub struct Tunables { /// /// If "None", nexus loops forever during initialization. pub load_timeout: Option, + + /// The amount of disk space to reserve for non-Crucible / control plane + /// storage in gibibytes. This amount represents a buffer that the region + /// allocation query will not use for each U2. + pub control_plane_storage_buffer_gb: u32, } // Convert from the unvalidated tunables, verifying each parameter as needed. @@ -314,6 +320,8 @@ impl TryFrom for Tunables { Ok(Tunables { max_vpc_ipv4_subnet_prefix: unvalidated.max_vpc_ipv4_subnet_prefix, load_timeout: unvalidated.load_timeout, + control_plane_storage_buffer_gb: unvalidated + .control_plane_storage_buffer_gb, }) } } @@ -365,6 +373,7 @@ impl Default for Tunables { Tunables { max_vpc_ipv4_subnet_prefix: MAX_VPC_IPV4_SUBNET_PREFIX, load_timeout: None, + control_plane_storage_buffer_gb: 0, } } } @@ -1014,6 +1023,7 @@ mod test { trusted_root = "/path/to/root.json" [tunables] max_vpc_ipv4_subnet_prefix = 27 + control_plane_storage_buffer_gb = 0 [deployment] id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f" @@ -1156,7 +1166,8 @@ mod test { schema: None, tunables: Tunables { max_vpc_ipv4_subnet_prefix: 27, - load_timeout: None + load_timeout: None, + control_plane_storage_buffer_gb: 0, }, dendrite: HashMap::from([( SwitchLocation::Switch0, @@ -1477,6 +1488,7 @@ mod test { default_base_url = "http://example.invalid/" [tunables] max_vpc_ipv4_subnet_prefix = 100 + control_plane_storage_buffer_gb = 0 [deployment] id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f" diff --git a/nexus/examples/config-second.toml b/nexus/examples/config-second.toml index 5387793ec81..eb6c9a988ec 100644 --- a/nexus/examples/config-second.toml +++ b/nexus/examples/config-second.toml @@ -86,6 +86,9 @@ default_request_body_max_bytes = 1048576 # IPv4 subnetwork. This size allows for ~60 hosts. max_vpc_ipv4_subnet_prefix = 26 +# For development environments, choose a zero sized storage buffer +control_plane_storage_buffer_gb = 0 + # Configuration for interacting with the dataplane daemon [dendrite.switch0] address = "[::1]:12224" diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index e5b4d564f55..fb70bf16c9a 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -72,6 +72,9 @@ url = "postgresql://root@[::1]:32221/omicron?sslmode=disable" # IPv4 subnetwork. This size allows for ~60 hosts. max_vpc_ipv4_subnet_prefix = 26 +# For development environments, choose a zero sized storage buffer +control_plane_storage_buffer_gb = 0 + # Configuration for interacting with the dataplane daemon [dendrite.switch0] address = "[::1]:12224" diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index 2da3b146a06..1b522ac76b2 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -137,6 +137,7 @@ use nexus_db_model::DnsGroup; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_types::deployment::PendingMgsUpdates; +use omicron_common::api::external::ByteCount; use omicron_uuid_kinds::OmicronZoneUuid; use oximeter::types::ProducerRegistry; use std::collections::BTreeMap; @@ -525,6 +526,7 @@ impl BackgroundTasksInitializer { inventory_watcher.clone(), config.physical_disk_adoption.disable, rack_id, + args.control_plane_storage_buffer, ), ), opctx: opctx.child(BTreeMap::new()), @@ -959,6 +961,10 @@ pub struct BackgroundTasksData { pub webhook_delivery_client: reqwest::Client, /// Channel for configuring pending MGS updates pub mgs_updates_tx: watch::Sender, + /// The amount of disk space to reserve for non-Crucible / control plane + /// storage. This amount represents a buffer that the region allocation query + /// will not use for each U2. + pub control_plane_storage_buffer: ByteCount, } /// Starts the three DNS-propagation-related background tasks for either diff --git a/nexus/src/app/background/tasks/physical_disk_adoption.rs b/nexus/src/app/background/tasks/physical_disk_adoption.rs index 35c72437ebc..e9d0c167914 100644 --- a/nexus/src/app/background/tasks/physical_disk_adoption.rs +++ b/nexus/src/app/background/tasks/physical_disk_adoption.rs @@ -11,7 +11,6 @@ //! //! In the future, this may become more explicitly operator-controlled. -use crate::app::CONTROL_PLANE_STORAGE_BUFFER; use crate::app::background::BackgroundTask; use futures::FutureExt; use futures::future::BoxFuture; @@ -20,6 +19,7 @@ use nexus_db_model::Zpool; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_types::identity::Asset; +use omicron_common::api::external::ByteCount; use omicron_common::api::external::DataPageParams; use omicron_uuid_kinds::CollectionUuid; use omicron_uuid_kinds::GenericUuid; @@ -34,6 +34,7 @@ pub struct PhysicalDiskAdoption { disable: bool, rack_id: Uuid, rx_inventory_collection: watch::Receiver>, + control_plane_storage_buffer: ByteCount, } impl PhysicalDiskAdoption { @@ -42,12 +43,14 @@ impl PhysicalDiskAdoption { rx_inventory_collection: watch::Receiver>, disable: bool, rack_id: Uuid, + control_plane_storage_buffer: ByteCount, ) -> Self { PhysicalDiskAdoption { datastore, disable, rack_id, rx_inventory_collection, + control_plane_storage_buffer, } } } @@ -140,7 +143,7 @@ impl BackgroundTask for PhysicalDiskAdoption { Uuid::new_v4(), inv_disk.sled_id.into_untyped_uuid(), disk.id(), - CONTROL_PLANE_STORAGE_BUFFER.into(), + self.control_plane_storage_buffer.into(), ); let result = self.datastore.physical_disk_and_zpool_insert( diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 1291f369c73..83faf9a4940 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -135,14 +135,6 @@ pub const MAX_DISK_SIZE_BYTES: u64 = 1023 * (1 << 30); // 1023 GiB /// This value is aribtrary pub const MAX_SSH_KEYS_PER_INSTANCE: u32 = 100; -/// The amount of disk space to reserve for non-Crucible / control plane -/// storage. This amount represents a buffer that the region allocation query -/// will not use for each U2. -/// -/// See oxidecomputer/omicron#7875 for the 250G determination. -pub const CONTROL_PLANE_STORAGE_BUFFER: ByteCount = - ByteCount::from_gibibytes_u32(250); - /// Manages an Oxide fleet -- the heart of the control plane pub struct Nexus { /// uuid for this nexus instance. @@ -253,6 +245,11 @@ pub struct Nexus { /// reports status of pending MGS-managed updates mgs_update_status_rx: watch::Receiver, + + /// The amount of disk space to reserve for non-Crucible / control plane + /// storage. This amount represents a buffer that the region allocation query + /// will not use for each U2. + control_plane_storage_buffer: ByteCount, } impl Nexus { @@ -429,6 +426,10 @@ impl Nexus { let mgs_update_status_rx = mgs_update_driver.status_rx(); let _mgs_driver_task = tokio::spawn(mgs_update_driver.run()); + let control_plane_storage_buffer = ByteCount::from_gibibytes_u32( + config.pkg.tunables.control_plane_storage_buffer_gb, + ); + let nexus = Nexus { id: config.deployment.id, rack_id, @@ -480,6 +481,7 @@ impl Nexus { )), tuf_artifact_replication_tx, mgs_update_status_rx, + control_plane_storage_buffer, }; // TODO-cleanup all the extra Arcs here seems wrong @@ -539,6 +541,7 @@ impl Nexus { }, tuf_artifact_replication_rx, mgs_updates_tx, + control_plane_storage_buffer, }, ); diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 3db5e46f967..69e9896fa4d 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -4,7 +4,6 @@ //! Rack management -use crate::app::CONTROL_PLANE_STORAGE_BUFFER; use crate::external_api::params; use crate::external_api::params::CertificateCreate; use crate::external_api::shared::ServiceUsingCertificate; @@ -137,7 +136,7 @@ impl super::Nexus { pool.id, pool.sled_id, pool.physical_disk_id, - CONTROL_PLANE_STORAGE_BUFFER.into(), + self.control_plane_storage_buffer.into(), ) }) .collect(); diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index ab3cbcd9af9..f003d7a5b13 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -34,6 +34,9 @@ address = "[::1]:0" # Allow small subnets, so we can test IP address exhaustion easily / quickly max_vpc_ipv4_subnet_prefix = 29 +# For development environments, choose a zero sized storage buffer +control_plane_storage_buffer_gb = 0 + [deployment] # Identifier for this instance of Nexus. # NOTE: The test suite always overrides this. diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml index 770bee73b1b..33041bff917 100644 --- a/smf/nexus/multi-sled/config-partial.toml +++ b/smf/nexus/multi-sled/config-partial.toml @@ -24,6 +24,13 @@ if_exists = "append" # [schema] # schema_dir = "/var/nexus/schema/crdb" +[tunables] +max_vpc_ipv4_subnet_prefix = 26 + +# Reserve space for non-Crucible storage +# See oxidecomputer/omicron#7875 for the 250G determination. +control_plane_storage_buffer_gb = 250 + [background_tasks] dns_internal.period_secs_config = 60 dns_internal.period_secs_servers = 60 diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml index eefe7554ed6..9471ed55429 100644 --- a/smf/nexus/single-sled/config-partial.toml +++ b/smf/nexus/single-sled/config-partial.toml @@ -24,6 +24,12 @@ if_exists = "append" # [schema] # schema_dir = "/var/nexus/schema/crdb" +[tunables] +max_vpc_ipv4_subnet_prefix = 26 + +# For development environments, choose a zero sized storage buffer +control_plane_storage_buffer_gb = 0 + [background_tasks] dns_internal.period_secs_config = 60 dns_internal.period_secs_servers = 60 From 647f2802f4b4b415721bac75e69836651337f964 Mon Sep 17 00:00:00 2001 From: James MacMahon Date: Wed, 7 May 2025 14:56:02 +0000 Subject: [PATCH 2/2] Revert change to configuration, and instead use settings Introduce idea of a database resident Nexus setting that is configured through rack initialization. This can be changed or removed later. Revert changes adding control plane storage buffer to tunables list. --- nexus-config/src/nexus_config.rs | 14 +-- nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/schema_versions.rs | 3 +- nexus/db-model/src/setting.rs | 23 +++++ nexus/db-queries/src/db/datastore/mod.rs | 1 + nexus/db-queries/src/db/datastore/rack.rs | 16 +++ nexus/db-queries/src/db/datastore/setting.rs | 97 +++++++++++++++++++ nexus/db-schema/src/enums.rs | 7 +- nexus/db-schema/src/schema.rs | 7 ++ nexus/examples/config-second.toml | 3 - nexus/examples/config.toml | 3 - nexus/src/app/background/init.rs | 6 -- .../tasks/physical_disk_adoption.rs | 24 ++++- nexus/src/app/mod.rs | 12 --- nexus/src/app/rack.rs | 8 +- nexus/src/lib.rs | 1 + nexus/tests/config.test.toml | 3 - nexus/types/src/internal_api/params.rs | 4 + openapi/bootstrap-agent.json | 7 ++ openapi/nexus-internal.json | 7 ++ schema/crdb/dbinit.sql | 12 ++- schema/crdb/nexus-settings/up01.sql | 3 + schema/crdb/nexus-settings/up02.sql | 4 + sled-agent/src/rack_setup/plan/service.rs | 1 + sled-agent/src/rack_setup/service.rs | 2 + sled-agent/src/sim/server.rs | 1 + sled-agent/types/src/rack_init.rs | 18 ++++ smf/nexus/multi-sled/config-partial.toml | 7 -- smf/nexus/single-sled/config-partial.toml | 6 -- wicketd/src/rss_config.rs | 4 + 30 files changed, 243 insertions(+), 63 deletions(-) create mode 100644 nexus/db-model/src/setting.rs create mode 100644 nexus/db-queries/src/db/datastore/setting.rs create mode 100644 schema/crdb/nexus-settings/up01.sql create mode 100644 schema/crdb/nexus-settings/up02.sql diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index e1a11d2275b..1364a52fbad 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -275,7 +275,6 @@ pub struct MgdConfig { struct UnvalidatedTunables { max_vpc_ipv4_subnet_prefix: u8, load_timeout: Option, - control_plane_storage_buffer_gb: u32, } /// Configuration for HTTP clients to external services. @@ -304,11 +303,6 @@ pub struct Tunables { /// /// If "None", nexus loops forever during initialization. pub load_timeout: Option, - - /// The amount of disk space to reserve for non-Crucible / control plane - /// storage in gibibytes. This amount represents a buffer that the region - /// allocation query will not use for each U2. - pub control_plane_storage_buffer_gb: u32, } // Convert from the unvalidated tunables, verifying each parameter as needed. @@ -320,8 +314,6 @@ impl TryFrom for Tunables { Ok(Tunables { max_vpc_ipv4_subnet_prefix: unvalidated.max_vpc_ipv4_subnet_prefix, load_timeout: unvalidated.load_timeout, - control_plane_storage_buffer_gb: unvalidated - .control_plane_storage_buffer_gb, }) } } @@ -373,7 +365,6 @@ impl Default for Tunables { Tunables { max_vpc_ipv4_subnet_prefix: MAX_VPC_IPV4_SUBNET_PREFIX, load_timeout: None, - control_plane_storage_buffer_gb: 0, } } } @@ -1023,7 +1014,6 @@ mod test { trusted_root = "/path/to/root.json" [tunables] max_vpc_ipv4_subnet_prefix = 27 - control_plane_storage_buffer_gb = 0 [deployment] id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f" @@ -1166,8 +1156,7 @@ mod test { schema: None, tunables: Tunables { max_vpc_ipv4_subnet_prefix: 27, - load_timeout: None, - control_plane_storage_buffer_gb: 0, + load_timeout: None }, dendrite: HashMap::from([( SwitchLocation::Switch0, @@ -1488,7 +1477,6 @@ mod test { default_base_url = "http://example.invalid/" [tunables] max_vpc_ipv4_subnet_prefix = 100 - control_plane_storage_buffer_gb = 0 [deployment] id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f" diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 3627e8c395c..0467610f2cb 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -95,6 +95,7 @@ mod role_builtin; pub mod saga_types; mod schema_versions; mod service_kind; +mod setting; mod silo; mod silo_group; mod silo_user; @@ -207,6 +208,7 @@ pub use saga_types::*; pub use schema_versions::*; pub use semver_version::*; pub use service_kind::*; +pub use setting::*; pub use silo::*; pub use silo_group::*; pub use silo_user::*; diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index d98bd3482f0..8b1bf80d5ac 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock}; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: Version = Version::new(140, 0, 0); +pub const SCHEMA_VERSION: Version = Version::new(141, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -28,6 +28,7 @@ static KNOWN_VERSIONS: LazyLock> = LazyLock::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(141, "nexus-settings"), KnownVersion::new(140, "instance-intended-state"), KnownVersion::new(139, "webhooks"), KnownVersion::new(138, "saga-abandoned-state"), diff --git a/nexus/db-model/src/setting.rs b/nexus/db-model/src/setting.rs new file mode 100644 index 00000000000..412ea3568d3 --- /dev/null +++ b/nexus/db-model/src/setting.rs @@ -0,0 +1,23 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::impl_enum_type; +use nexus_db_schema::schema::setting; + +impl_enum_type!( + SettingNameEnum: + + #[derive(Copy, Clone, Debug, AsExpression, FromSqlRow, PartialEq)] + pub enum SettingName; + + // Enum values + ControlPlaneStorageBuffer => b"control_plane_storage_buffer" +); + +#[derive(Queryable, Insertable, Debug, Selectable, Clone)] +#[diesel(table_name = setting)] +pub struct Setting { + pub name: SettingName, + pub int_value: Option, +} diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 5a2acb6aff5..e6dff4b7cf7 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -87,6 +87,7 @@ pub mod region_snapshot_replacement; mod rendezvous_debug_dataset; mod role; mod saga; +mod setting; mod silo; mod silo_group; mod silo_user; diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 3236abc857b..24ad4e3d068 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -90,6 +90,7 @@ pub struct RackInit { pub recovery_user_password_hash: omicron_passwords::PasswordHashString, pub dns_update: DnsVersionUpdateBuilder, pub allowed_source_ips: AllowedSourceIps, + pub control_plane_storage_buffer_gib: u32, } /// Possible errors while trying to initialize rack @@ -112,6 +113,8 @@ enum RackInitError { Database(DieselError), // Error adding initial allowed source IP list AllowedSourceIpError(Error), + // Error changing a Nexus setting + ChangeSetting(Error), } // Catch-all for Diesel error conversion into RackInitError, which @@ -175,6 +178,7 @@ impl From for Error { err )), RackInitError::AllowedSourceIpError(err) => err, + RackInitError::ChangeSetting(err) => err, } } } @@ -911,6 +915,17 @@ impl DataStore { DieselError::RollbackTransaction })?; + Self::set_control_plane_storage_buffer_gib_impl( + opctx, + &conn, + rack_init.control_plane_storage_buffer_gib, + ) + .await + .map_err(|e| { + err.set(RackInitError::ChangeSetting(e)).unwrap(); + DieselError::RollbackTransaction + })?; + let rack = diesel::update(rack_dsl::rack) .filter(rack_dsl::id.eq(rack_id)) .set(( @@ -1117,6 +1132,7 @@ mod test { "test suite".to_string(), ), allowed_source_ips: AllowedSourceIps::Any, + control_plane_storage_buffer_gib: 0, } } } diff --git a/nexus/db-queries/src/db/datastore/setting.rs b/nexus/db-queries/src/db/datastore/setting.rs new file mode 100644 index 00000000000..e9ac846ddac --- /dev/null +++ b/nexus/db-queries/src/db/datastore/setting.rs @@ -0,0 +1,97 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! [`DataStore`] methods on [`Setting`]s. + +// Settings are dynamically controlled values for Nexus + +use super::DataStore; +use crate::authz; +use crate::context::OpContext; +use crate::db::datastore::ErrorHandler; +use crate::db::datastore::public_error_from_diesel; +use async_bb8_diesel::AsyncRunQueryDsl; +use diesel::prelude::*; +use nexus_db_lookup::DbConnection; +use nexus_db_model::Setting; +use nexus_db_model::SettingName; +use omicron_common::api::external::ByteCount; +use omicron_common::api::external::Error; + +impl DataStore { + pub(crate) async fn set_control_plane_storage_buffer_gib_impl( + opctx: &OpContext, + conn: &async_bb8_diesel::Connection, + gibibytes: u32, + ) -> Result<(), Error> { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + + use nexus_db_schema::schema::setting::dsl; + + let name = SettingName::ControlPlaneStorageBuffer; + let value: i64 = ByteCount::from_gibibytes_u32(gibibytes).into(); + + let maybe_existing = + Self::get_control_plane_storage_buffer_impl(opctx, conn).await?; + + if maybe_existing.is_some() { + diesel::update(dsl::setting) + .filter(dsl::name.eq(name)) + .set(dsl::int_value.eq(value)) + .execute_async(conn) + .await + .map(|_| ()) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } else { + let setting = Setting { name, int_value: Some(value) }; + + diesel::insert_into(dsl::setting) + .values(setting) + .execute_async(conn) + .await + .map(|_| ()) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + } + + pub(crate) async fn get_control_plane_storage_buffer_impl( + opctx: &OpContext, + conn: &async_bb8_diesel::Connection, + ) -> Result, Error> { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + + use nexus_db_schema::schema::setting::dsl; + + let name = SettingName::ControlPlaneStorageBuffer; + + let result = dsl::setting + .filter(dsl::name.eq(name)) + .select(Setting::as_select()) + .first_async(conn) + .await + .optional() + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(match result { + Some(setting) => { + // A row exists with this setting's name + match setting.int_value { + Some(value) => Some(ByteCount::try_from(value).unwrap()), + + None => None, + } + } + + None => None, + }) + } + + pub async fn get_control_plane_storage_buffer( + &self, + opctx: &OpContext, + ) -> Result, Error> { + let conn = self.pool_connection_authorized(opctx).await?; + Self::get_control_plane_storage_buffer_impl(opctx, &conn).await + } +} diff --git a/nexus/db-schema/src/enums.rs b/nexus/db-schema/src/enums.rs index 79d312c8b64..b4d59582461 100644 --- a/nexus/db-schema/src/enums.rs +++ b/nexus/db-schema/src/enums.rs @@ -41,8 +41,8 @@ define_enums! { IdentityProviderTypeEnum => "provider_type", IdentityTypeEnum => "identity_type", InstanceAutoRestartPolicyEnum => "instance_auto_restart", - InstanceStateEnum => "instance_state_v2", InstanceIntendedStateEnum => "instance_intended_state", + InstanceStateEnum => "instance_state_v2", IpAttachStateEnum => "ip_attach_state", IpKindEnum => "ip_kind", IpPoolResourceTypeEnum => "ip_pool_resource_type", @@ -64,6 +64,7 @@ define_enums! { RouterRouteKindEnum => "router_route_kind", SagaStateEnum => "saga_state", ServiceKindEnum => "service_kind", + SettingNameEnum => "setting_name", SledPolicyEnum => "sled_policy", SledRoleEnum => "sled_role", SledStateEnum => "sled_state", @@ -85,9 +86,9 @@ define_enums! { VpcFirewallRuleProtocolEnum => "vpc_firewall_rule_protocol", VpcFirewallRuleStatusEnum => "vpc_firewall_rule_status", VpcRouterKindEnum => "vpc_router_kind", - WebhookEventClassEnum => "webhook_event_class", WebhookDeliveryAttemptResultEnum => "webhook_delivery_attempt_result", - WebhookDeliveryTriggerEnum => "webhook_delivery_trigger", WebhookDeliveryStateEnum => "webhook_delivery_state", + WebhookDeliveryTriggerEnum => "webhook_delivery_trigger", + WebhookEventClassEnum => "webhook_event_class", ZoneTypeEnum => "zone_type", } diff --git a/nexus/db-schema/src/schema.rs b/nexus/db-schema/src/schema.rs index ba699b579ac..5e2eeb07de4 100644 --- a/nexus/db-schema/src/schema.rs +++ b/nexus/db-schema/src/schema.rs @@ -2309,3 +2309,10 @@ allow_tables_to_appear_in_same_query!( webhook_delivery_attempt ); joinable!(webhook_delivery_attempt -> webhook_delivery (delivery_id)); + +table! { + setting (name) { + name -> crate::enums::SettingNameEnum, + int_value -> Nullable, + } +} diff --git a/nexus/examples/config-second.toml b/nexus/examples/config-second.toml index eb6c9a988ec..5387793ec81 100644 --- a/nexus/examples/config-second.toml +++ b/nexus/examples/config-second.toml @@ -86,9 +86,6 @@ default_request_body_max_bytes = 1048576 # IPv4 subnetwork. This size allows for ~60 hosts. max_vpc_ipv4_subnet_prefix = 26 -# For development environments, choose a zero sized storage buffer -control_plane_storage_buffer_gb = 0 - # Configuration for interacting with the dataplane daemon [dendrite.switch0] address = "[::1]:12224" diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index fb70bf16c9a..e5b4d564f55 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -72,9 +72,6 @@ url = "postgresql://root@[::1]:32221/omicron?sslmode=disable" # IPv4 subnetwork. This size allows for ~60 hosts. max_vpc_ipv4_subnet_prefix = 26 -# For development environments, choose a zero sized storage buffer -control_plane_storage_buffer_gb = 0 - # Configuration for interacting with the dataplane daemon [dendrite.switch0] address = "[::1]:12224" diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index 1b522ac76b2..2da3b146a06 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -137,7 +137,6 @@ use nexus_db_model::DnsGroup; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_types::deployment::PendingMgsUpdates; -use omicron_common::api::external::ByteCount; use omicron_uuid_kinds::OmicronZoneUuid; use oximeter::types::ProducerRegistry; use std::collections::BTreeMap; @@ -526,7 +525,6 @@ impl BackgroundTasksInitializer { inventory_watcher.clone(), config.physical_disk_adoption.disable, rack_id, - args.control_plane_storage_buffer, ), ), opctx: opctx.child(BTreeMap::new()), @@ -961,10 +959,6 @@ pub struct BackgroundTasksData { pub webhook_delivery_client: reqwest::Client, /// Channel for configuring pending MGS updates pub mgs_updates_tx: watch::Sender, - /// The amount of disk space to reserve for non-Crucible / control plane - /// storage. This amount represents a buffer that the region allocation query - /// will not use for each U2. - pub control_plane_storage_buffer: ByteCount, } /// Starts the three DNS-propagation-related background tasks for either diff --git a/nexus/src/app/background/tasks/physical_disk_adoption.rs b/nexus/src/app/background/tasks/physical_disk_adoption.rs index e9d0c167914..d01da775a47 100644 --- a/nexus/src/app/background/tasks/physical_disk_adoption.rs +++ b/nexus/src/app/background/tasks/physical_disk_adoption.rs @@ -34,7 +34,6 @@ pub struct PhysicalDiskAdoption { disable: bool, rack_id: Uuid, rx_inventory_collection: watch::Receiver>, - control_plane_storage_buffer: ByteCount, } impl PhysicalDiskAdoption { @@ -43,14 +42,12 @@ impl PhysicalDiskAdoption { rx_inventory_collection: watch::Receiver>, disable: bool, rack_id: Uuid, - control_plane_storage_buffer: ByteCount, ) -> Self { PhysicalDiskAdoption { datastore, disable, rack_id, rx_inventory_collection, - control_plane_storage_buffer, } } } @@ -65,6 +62,25 @@ impl BackgroundTask for PhysicalDiskAdoption { return json!({ "error": "task disabled" }); } + let control_plane_storage_buffer: ByteCount = match self + .datastore + .get_control_plane_storage_buffer(opctx) + .await { + Ok(value) => match value { + Some(value) => value, + None => { + // If no setting is in the database, use 0 + ByteCount::from(0) + } + }, + + Err(e) => { + return json!({ "error": format!( + "error getting control plane storage buffer: {e}" + )}); + } + }; + // Only adopt physical disks after rack handoff has completed. // // This prevents a race condition where the same physical disks @@ -143,7 +159,7 @@ impl BackgroundTask for PhysicalDiskAdoption { Uuid::new_v4(), inv_disk.sled_id.into_untyped_uuid(), disk.id(), - self.control_plane_storage_buffer.into(), + control_plane_storage_buffer.into(), ); let result = self.datastore.physical_disk_and_zpool_insert( diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 83faf9a4940..0f6609a5c89 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -31,7 +31,6 @@ use nexus_types::deployment::PendingMgsUpdates; use omicron_common::address::DENDRITE_PORT; use omicron_common::address::MGD_PORT; use omicron_common::address::MGS_PORT; -use omicron_common::api::external::ByteCount; use omicron_common::api::external::Error; use omicron_common::api::internal::shared::SwitchLocation; use omicron_uuid_kinds::OmicronZoneUuid; @@ -245,11 +244,6 @@ pub struct Nexus { /// reports status of pending MGS-managed updates mgs_update_status_rx: watch::Receiver, - - /// The amount of disk space to reserve for non-Crucible / control plane - /// storage. This amount represents a buffer that the region allocation query - /// will not use for each U2. - control_plane_storage_buffer: ByteCount, } impl Nexus { @@ -426,10 +420,6 @@ impl Nexus { let mgs_update_status_rx = mgs_update_driver.status_rx(); let _mgs_driver_task = tokio::spawn(mgs_update_driver.run()); - let control_plane_storage_buffer = ByteCount::from_gibibytes_u32( - config.pkg.tunables.control_plane_storage_buffer_gb, - ); - let nexus = Nexus { id: config.deployment.id, rack_id, @@ -481,7 +471,6 @@ impl Nexus { )), tuf_artifact_replication_tx, mgs_update_status_rx, - control_plane_storage_buffer, }; // TODO-cleanup all the extra Arcs here seems wrong @@ -541,7 +530,6 @@ impl Nexus { }, tuf_artifact_replication_rx, mgs_updates_tx, - control_plane_storage_buffer, }, ); diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 69e9896fa4d..910028dfdb1 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -52,6 +52,7 @@ use nexus_types::silo::silo_dns_name; use omicron_common::address::{Ipv6Subnet, RACK_PREFIX, get_64_subnet}; use omicron_common::api::external::AddressLotKind; use omicron_common::api::external::BgpPeer; +use omicron_common::api::external::ByteCount; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::IdentityMetadataCreateParams; @@ -136,7 +137,10 @@ impl super::Nexus { pool.id, pool.sled_id, pool.physical_disk_id, - self.control_plane_storage_buffer.into(), + ByteCount::from_gibibytes_u32( + request.control_plane_storage_buffer_gib, + ) + .into(), ) }) .collect(); @@ -727,6 +731,8 @@ impl super::Nexus { .into(), dns_update, allowed_source_ips: request.allowed_source_ips, + control_plane_storage_buffer_gib: request + .control_plane_storage_buffer_gib, }, ) .await?; diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index a2ee375726a..011b85a23f7 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -330,6 +330,7 @@ impl nexus_test_interface::NexusServer for Server { bfd: Vec::new(), }, allowed_source_ips: AllowedSourceIps::Any, + control_plane_storage_buffer_gib: 0, }, ) .await diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index f003d7a5b13..ab3cbcd9af9 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -34,9 +34,6 @@ address = "[::1]:0" # Allow small subnets, so we can test IP address exhaustion easily / quickly max_vpc_ipv4_subnet_prefix = 29 -# For development environments, choose a zero sized storage buffer -control_plane_storage_buffer_gb = 0 - [deployment] # Identifier for this instance of Nexus. # NOTE: The test suite always overrides this. diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs index a1a707d12a9..70823656ea5 100644 --- a/nexus/types/src/internal_api/params.rs +++ b/nexus/types/src/internal_api/params.rs @@ -189,6 +189,10 @@ pub struct RackInitializationRequest { pub rack_network_config: RackNetworkConfig, /// IPs or subnets allowed to make requests to user-facing services pub allowed_source_ips: AllowedSourceIps, + /// The amount of space to reserve per-disk for non-Crucible storage (i.e. + /// control plane storage) in gibibytes. This amount represents a buffer + /// that the region allocation query will not use for each U2. + pub control_plane_storage_buffer_gib: u32, } pub type DnsConfigParams = internal_dns_types::config::DnsConfigParams; diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index ce16f185800..49bfa887532 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -944,6 +944,13 @@ } ] }, + "control_plane_storage_buffer_gib": { + "description": "The amount of space to reserve per-disk for non-Crucible storage (i.e. control plane storage) in gibibytes. This amount represents a buffer that the region allocation query will not use for each U2.", + "default": 0, + "type": "integer", + "format": "uint32", + "minimum": 0 + }, "dns_servers": { "description": "The external DNS server addresses.", "type": "array", diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index a9c6d4fa074..a42851569a1 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -5778,6 +5778,12 @@ "$ref": "#/components/schemas/Certificate" } }, + "control_plane_storage_buffer_gib": { + "description": "The amount of space to reserve per-disk for non-Crucible storage (i.e. control plane storage) in gibibytes. This amount represents a buffer that the region allocation query will not use for each U2.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, "crucible_datasets": { "description": "Crucible datasets on the rack which have been provisioned by RSS.", "type": "array", @@ -5847,6 +5853,7 @@ "allowed_source_ips", "blueprint", "certs", + "control_plane_storage_buffer_gib", "crucible_datasets", "external_dns_zone_name", "external_port_count", diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index d0b0e28004f..c58133f1743 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5493,6 +5493,16 @@ ON omicron.public.webhook_delivery_attempt ( rx_id ); +CREATE TYPE IF NOT EXISTS omicron.public.setting_name AS ENUM ( + 'control_plane_storage_buffer' +); + +/* A table of Nexus' dynamic settings */ +CREATE TABLE IF NOT EXISTS omicron.public.setting ( + name omicron.public.setting_name PRIMARY KEY, + int_value INT +); + /* * Keep this at the end of file so that the database does not contain a version * until it is fully populated. @@ -5504,7 +5514,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '140.0.0', NULL) + (TRUE, NOW(), NOW(), '141.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/nexus-settings/up01.sql b/schema/crdb/nexus-settings/up01.sql new file mode 100644 index 00000000000..1821e337b30 --- /dev/null +++ b/schema/crdb/nexus-settings/up01.sql @@ -0,0 +1,3 @@ +CREATE TYPE IF NOT EXISTS omicron.public.setting_name AS ENUM ( + 'control_plane_storage_buffer' +); diff --git a/schema/crdb/nexus-settings/up02.sql b/schema/crdb/nexus-settings/up02.sql new file mode 100644 index 00000000000..9251b1d9f73 --- /dev/null +++ b/schema/crdb/nexus-settings/up02.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS omicron.public.setting ( + name omicron.public.setting_name PRIMARY KEY, + int_value INT +); diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 3a58e9583a8..a05b70bb332 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -1253,6 +1253,7 @@ mod tests { bfd: Vec::new(), }, allowed_source_ips: AllowedSourceIps::Any, + control_plane_storage_buffer_gib: 0, } } diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 85bef7032c6..b1da9e41dbd 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -988,6 +988,8 @@ impl ServiceInner { rack_network_config, external_port_count: port_discovery_mode.into(), allowed_source_ips, + control_plane_storage_buffer_gib: config + .control_plane_storage_buffer_gib, }; let notify_nexus = || async { diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index 05c75e18c0e..8d0bbce9ef2 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -595,6 +595,7 @@ pub async fn run_standalone_server( bfd: Vec::new(), }, allowed_source_ips: NexusTypes::AllowedSourceIps::Any, + control_plane_storage_buffer_gib: 0, }; handoff_to_nexus(&log, &config, &rack_init_request).await?; diff --git a/sled-agent/types/src/rack_init.rs b/sled-agent/types/src/rack_init.rs index 0f047734c10..54a83cc610d 100644 --- a/sled-agent/types/src/rack_init.rs +++ b/sled-agent/types/src/rack_init.rs @@ -110,6 +110,7 @@ pub mod back_compat { recovery_silo: v1.recovery_silo, rack_network_config: v1.rack_network_config.into(), allowed_source_ips: v1.allowed_source_ips, + control_plane_storage_buffer_gib: 0, } } } @@ -131,6 +132,8 @@ struct UnvalidatedRackInitializeRequest { rack_network_config: RackNetworkConfig, #[serde(default = "default_allowed_source_ips")] allowed_source_ips: AllowedSourceIps, + #[serde(default)] + control_plane_storage_buffer_gib: u32, } fn validate_external_dns( @@ -177,6 +180,8 @@ impl TryFrom for RackInitializeRequest { recovery_silo: value.recovery_silo, rack_network_config: value.rack_network_config, allowed_source_ips: value.allowed_source_ips, + control_plane_storage_buffer_gib: value + .control_plane_storage_buffer_gib, }) } } @@ -229,6 +234,12 @@ pub struct RackInitializeRequest { /// IPs or subnets allowed to make requests to user-facing services #[serde(default = "default_allowed_source_ips")] pub allowed_source_ips: AllowedSourceIps, + + /// The amount of space to reserve per-disk for non-Crucible storage (i.e. + /// control plane storage) in gibibytes. This amount represents a buffer + /// that the region allocation query will not use for each U2. + #[serde(default)] + pub control_plane_storage_buffer_gib: u32, } impl RackInitializeRequest { @@ -386,6 +397,7 @@ impl std::fmt::Debug for RackInitializeRequest { recovery_silo, rack_network_config, allowed_source_ips, + control_plane_storage_buffer_gib, } = &self; f.debug_struct("RackInitializeRequest") @@ -403,6 +415,10 @@ impl std::fmt::Debug for RackInitializeRequest { .field("recovery_silo", recovery_silo) .field("rack_network_config", rack_network_config) .field("allowed_source_ips", allowed_source_ips) + .field( + "control_plane_storage_buffer_gib", + control_plane_storage_buffer_gib, + ) .finish() } } @@ -502,6 +518,7 @@ mod tests { bfd: Vec::new(), }, allowed_source_ips: AllowedSourceIps::Any, + control_plane_storage_buffer_gib: 0, }; // Valid configs: all external DNS IPs are contained in the IP pool @@ -631,6 +648,7 @@ mod tests { bfd: Vec::new(), }, allowed_source_ips: AllowedSourceIps::Any, + control_plane_storage_buffer_gib: 0, }; assert_eq!( diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml index 33041bff917..770bee73b1b 100644 --- a/smf/nexus/multi-sled/config-partial.toml +++ b/smf/nexus/multi-sled/config-partial.toml @@ -24,13 +24,6 @@ if_exists = "append" # [schema] # schema_dir = "/var/nexus/schema/crdb" -[tunables] -max_vpc_ipv4_subnet_prefix = 26 - -# Reserve space for non-Crucible storage -# See oxidecomputer/omicron#7875 for the 250G determination. -control_plane_storage_buffer_gb = 250 - [background_tasks] dns_internal.period_secs_config = 60 dns_internal.period_secs_servers = 60 diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml index 9471ed55429..eefe7554ed6 100644 --- a/smf/nexus/single-sled/config-partial.toml +++ b/smf/nexus/single-sled/config-partial.toml @@ -24,12 +24,6 @@ if_exists = "append" # [schema] # schema_dir = "/var/nexus/schema/crdb" -[tunables] -max_vpc_ipv4_subnet_prefix = 26 - -# For development environments, choose a zero sized storage buffer -control_plane_storage_buffer_gb = 0 - [background_tasks] dns_internal.period_secs_config = 60 dns_internal.period_secs_servers = 60 diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs index 0223a565dfa..30d21228e86 100644 --- a/wicketd/src/rss_config.rs +++ b/wicketd/src/rss_config.rs @@ -314,6 +314,10 @@ impl CurrentRssConfig { .allowed_source_ips .clone() .unwrap_or(AllowedSourceIps::Any), + // Reserve a set amount of space for non-Crucible (i.e. control + // plane) storage. See oxidecomputer/omicron#7875 for the size + // determination. + control_plane_storage_buffer_gib: 250, }; Ok(request)