From 1e552110b43fae3a14207799f586eb6f15ea3bdf Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Fri, 17 Oct 2025 22:03:33 +0000 Subject: [PATCH] Add database support for dual-stack network interfaces - Add `ipv6` column to the `network_interface` table, and make both it and the `ip` column nullable to account for interfaces with one or both address families. - Add schema update files that recreate a bunch of views / indexes to account for the new columns. - Add a check constraint that there is at least one IP address, and update the code handling IP address exhaustion with that new constraint failure. - Closes #9242 --- dev-tools/omdb/src/bin/omdb/db.rs | 33 +++-- nexus/db-model/src/ipv4.rs | 92 +++++++++++++ nexus/db-model/src/ipv6.rs | 24 ++++ nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/network_interface.rs | 122 ++++++++++++------ nexus/db-model/src/schema_versions.rs | 3 +- nexus/db-model/src/v2p_mapping.rs | 17 ++- .../db-queries/src/db/datastore/deployment.rs | 2 +- .../deployment/external_networking.rs | 39 +++++- .../src/db/datastore/network_interface.rs | 87 +++++++------ nexus/db-queries/src/db/datastore/rack.rs | 23 +++- .../src/db/datastore/v2p_mapping.rs | 6 +- nexus/db-queries/src/db/datastore/vpc.rs | 23 ++-- .../src/db/queries/network_interface.rs | 72 +++++++---- nexus/db-schema/src/schema.rs | 12 +- .../src/app/background/tasks/v2p_mappings.rs | 26 +++- schema/crdb/dbinit.sql | 74 ++++++++--- .../dual-stack-network-interfaces/up01.sql | 2 + .../dual-stack-network-interfaces/up02.sql | 2 + .../dual-stack-network-interfaces/up03.sql | 2 + .../dual-stack-network-interfaces/up04.sql | 2 + .../dual-stack-network-interfaces/up05.sql | 2 + .../dual-stack-network-interfaces/up06.sql | 4 + .../dual-stack-network-interfaces/up07.sql | 5 + .../dual-stack-network-interfaces/up08.sql | 6 + .../dual-stack-network-interfaces/up09.sql | 21 +++ .../dual-stack-network-interfaces/up10.sql | 22 ++++ .../dual-stack-network-interfaces/up11.sql | 9 ++ .../dual-stack-network-interfaces/up12.sql | 8 ++ .../dual-stack-network-interfaces/up13.sql | 5 + .../dual-stack-network-interfaces/up14.sql | 6 + 31 files changed, 590 insertions(+), 163 deletions(-) create mode 100644 nexus/db-model/src/ipv4.rs create mode 100644 schema/crdb/dual-stack-network-interfaces/up01.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up02.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up03.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up04.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up05.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up06.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up07.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up08.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up09.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up10.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up11.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up12.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up13.sql create mode 100644 schema/crdb/dual-stack-network-interfaces/up14.sql diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index d4833c6d187..fe9d81cfce2 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -59,7 +59,6 @@ use indicatif::ProgressBar; use indicatif::ProgressDrawTarget; use indicatif::ProgressStyle; use internal_dns_types::names::ServiceName; -use ipnetwork::IpNetwork; use nexus_config::PostgresConfigWithUrl; use nexus_config::RegionAllocationStrategy; use nexus_db_errors::OptionalError; @@ -168,6 +167,8 @@ use std::collections::HashMap; use std::collections::HashSet; use std::fmt::Display; use std::future::Future; +use std::net::Ipv4Addr; +use std::net::Ipv6Addr; use std::num::NonZeroU32; use std::str::FromStr; use std::sync::Arc; @@ -5376,12 +5377,16 @@ async fn cmd_db_network_list_vnics( #[derive(Tabled)] #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] struct NicRow { - ip: IpNetwork, + #[tabled(display_with = "option_impl_display")] + ipv4: Option, + #[tabled(display_with = "option_impl_display")] + ipv6: Option, mac: MacAddr, slot: u8, primary: bool, kind: &'static str, - subnet: String, + ipv4_subnet: String, + ipv6_subnet: String, parent_id: Uuid, parent_name: String, } @@ -5471,7 +5476,7 @@ async fn cmd_db_network_list_vnics( } }; - let subnet = { + let (ipv4_subnet, ipv6_subnet) = { use nexus_db_schema::schema::vpc_subnet::dsl; let subnet = match dsl::vpc_subnet .filter(dsl::id.eq(nic.subnet_id)) @@ -5488,28 +5493,36 @@ async fn cmd_db_network_list_vnics( continue; } }; - - if nic.ip.is_ipv4() { + let ipv4_subnet = if nic.ipv4.is_some() { subnet.ipv4_block.to_string() } else { + String::from("-") + }; + let ipv6_subnet = if nic.ipv6.is_some() { subnet.ipv6_block.to_string() - } + } else { + String::from("-") + }; + (ipv4_subnet, ipv6_subnet) }; let row = NicRow { - ip: nic.ip, + ipv4: nic.ipv4.map(Into::into), + ipv6: nic.ipv6.map(Into::into), mac: *nic.mac, slot: *nic.slot, primary: nic.primary, kind, - subnet, + ipv4_subnet, + ipv6_subnet, parent_id: nic.parent_id, parent_name, }; rows.push(row); } - rows.sort_by(|a, b| a.ip.cmp(&b.ip)); + // Sort by IPv4 address, and then IPv6 address. + rows.sort_by(|a, b| a.ipv4.cmp(&b.ipv4).then_with(|| a.ipv6.cmp(&b.ipv6))); let table = tabled::Table::new(rows) .with(tabled::settings::Style::empty()) .to_string(); diff --git a/nexus/db-model/src/ipv4.rs b/nexus/db-model/src/ipv4.rs new file mode 100644 index 00000000000..a416f13ce07 --- /dev/null +++ b/nexus/db-model/src/ipv4.rs @@ -0,0 +1,92 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Database-friendly IPv4 addresses + +use diesel::backend::Backend; +use diesel::deserialize; +use diesel::deserialize::FromSql; +use diesel::pg::Pg; +use diesel::serialize; +use diesel::serialize::Output; +use diesel::serialize::ToSql; +use diesel::sql_types::Inet; +use ipnetwork::IpNetwork; +use ipnetwork::Ipv4Network; +use omicron_common::api::external::Error; +use serde::Deserialize; +use serde::Serialize; + +#[derive( + Clone, + Copy, + AsExpression, + FromSqlRow, + PartialEq, + Ord, + PartialOrd, + Eq, + Deserialize, + Serialize, +)] +#[diesel(sql_type = Inet)] +pub struct Ipv4Addr(std::net::Ipv4Addr); + +NewtypeDebug! { () pub struct Ipv4Addr(std::net::Ipv4Addr); } +NewtypeFrom! { () pub struct Ipv4Addr(std::net::Ipv4Addr); } +NewtypeDeref! { () pub struct Ipv4Addr(std::net::Ipv4Addr); } + +impl From<&std::net::Ipv4Addr> for Ipv4Addr { + fn from(addr: &std::net::Ipv4Addr) -> Self { + Self(*addr) + } +} + +impl From for std::net::IpAddr { + fn from(value: Ipv4Addr) -> Self { + std::net::IpAddr::from(value.0) + } +} + +impl From<&Ipv4Addr> for std::net::IpAddr { + fn from(value: &Ipv4Addr) -> Self { + (*value).into() + } +} + +impl From for Ipv4Network { + fn from(value: Ipv4Addr) -> Self { + Ipv4Network::from(value.0) + } +} + +impl From for IpNetwork { + fn from(value: Ipv4Addr) -> Self { + IpNetwork::V4(Ipv4Network::from(value.0)) + } +} + +impl ToSql for Ipv4Addr { + fn to_sql<'a>(&'a self, out: &mut Output<'a, '_, Pg>) -> serialize::Result { + let net = IpNetwork::V4(Ipv4Network::from(self.0)); + >::to_sql(&net, &mut out.reborrow()) + } +} + +impl FromSql for Ipv4Addr +where + DB: Backend, + IpNetwork: FromSql, +{ + fn from_sql(bytes: DB::RawValue<'_>) -> deserialize::Result { + match IpNetwork::from_sql(bytes)?.ip() { + std::net::IpAddr::V4(ip) => Ok(Self(ip)), + v6 => { + Err(Box::new(Error::internal_error( + format!("Expected an IPv4 address from the database, found IPv6: '{}'", v6).as_str() + ))) + } + } + } +} diff --git a/nexus/db-model/src/ipv6.rs b/nexus/db-model/src/ipv6.rs index d4935b3a61a..84b380abddc 100644 --- a/nexus/db-model/src/ipv6.rs +++ b/nexus/db-model/src/ipv6.rs @@ -42,6 +42,30 @@ impl From<&std::net::Ipv6Addr> for Ipv6Addr { } } +impl From for std::net::IpAddr { + fn from(value: Ipv6Addr) -> Self { + value.0.into() + } +} + +impl From<&Ipv6Addr> for std::net::IpAddr { + fn from(value: &Ipv6Addr) -> Self { + (*value).into() + } +} + +impl From for Ipv6Network { + fn from(value: Ipv6Addr) -> Self { + Ipv6Network::from(value.0) + } +} + +impl From for IpNetwork { + fn from(value: Ipv6Addr) -> Self { + IpNetwork::V6(Ipv6Network::from(value.0)) + } +} + impl ToSql for Ipv6Addr { fn to_sql<'a>(&'a self, out: &mut Output<'a, '_, Pg>) -> serialize::Result { let net = IpNetwork::V6(Ipv6Network::from(self.0)); diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index baa1a408407..02870a6439e 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -51,6 +51,7 @@ mod internet_gateway; mod inventory; mod ip_pool; mod ipnet; +pub mod ipv4; mod ipv4net; pub mod ipv6; mod ipv6net; @@ -192,6 +193,7 @@ pub use internet_gateway::*; pub use inventory::*; pub use ip_pool::*; pub use ipnet::*; +pub use ipv4::*; pub use ipv4net::*; pub use ipv6::*; pub use ipv6net::*; diff --git a/nexus/db-model/src/network_interface.rs b/nexus/db-model/src/network_interface.rs index 9f761cb7a87..de315f5ac79 100644 --- a/nexus/db-model/src/network_interface.rs +++ b/nexus/db-model/src/network_interface.rs @@ -3,6 +3,8 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use super::{MacAddr, VpcSubnet}; +use crate::Ipv4Addr; +use crate::Ipv6Addr; use crate::Name; use crate::SqlU8; use crate::impl_enum_type; @@ -11,13 +13,13 @@ use chrono::Utc; use db_macros::Resource; use diesel::AsChangeset; use ipnetwork::IpNetwork; -use ipnetwork::NetworkSize; use nexus_db_schema::schema::instance_network_interface; use nexus_db_schema::schema::network_interface; use nexus_db_schema::schema::service_network_interface; use nexus_sled_agent_shared::inventory::ZoneKind; use nexus_types::external_api::params; use nexus_types::identity::Resource; +use omicron_common::api::external::Error; use omicron_common::api::{external, internal}; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; @@ -49,25 +51,48 @@ impl_enum_type! { pub struct NetworkInterface { #[diesel(embed)] pub identity: NetworkInterfaceIdentity, - + /// Which kind of parent this interface belongs to. pub kind: NetworkInterfaceKind, + /// UUID of the parent. pub parent_id: Uuid, - + /// UUID of the VPC containing this interface. pub vpc_id: Uuid, + /// UUID of the VPC Subnet containing this interface. pub subnet_id: Uuid, - + /// MAC address for this interface. pub mac: MacAddr, - // TODO-correctness: We need to split this into an optional V4 and optional V6 address, at - // least one of which will always be specified. + /// The VPC-private IPv4 address of the interface. + /// + /// At least one of the `ip` and `ipv6` fields will always be `Some(_)`, a + /// constraint enforced by the database. Both may be `Some(_)` for + /// dual-stack interfaces. + // NOTE: At least one of the below will be non-None. // - // If user requests an address of either kind, give exactly that and not the other. - // If neither is specified, auto-assign one of each? - pub ip: IpNetwork, - + // We could use an enum to enforce this, but there's a lot of diesel + // machinery needed and it makes sharing the type between this model and the + // `InstanceNetworkInterface` below difficult. In particular, the db-lookup + // stuff chokes because we can't make the same type selectable from two + // different tables. In any case, we want to enforce this on the + // `IncompleteNetworkInterface` type, and we already do enforce it via a + // check constraint in the database itself. + // + // NOTE: The column in the database is still named `ip`, because renaming + // columns isn't idempotent in CRDB as of today. + #[diesel(column_name = ip)] + pub ipv4: Option, + /// The VPC-private IPv6 address of the interface. + /// + /// At least one of the `ip` and `ipv6` fields will always be `Some(_)`, a + /// constraint enforced by the database. Both may be `Some(_)` for + /// dual-stack interfaces. + pub ipv6: Option, + /// The PCI slot on the instance where the interface appears. pub slot: SqlU8, + /// True if this is the instance's primary interface. #[diesel(column_name = is_primary)] pub primary: bool, - + /// List of additional networks on which the instance is allowed to send / + /// receive traffic. pub transit_ips: Vec, } @@ -76,6 +101,12 @@ impl NetworkInterface { self, subnet: oxnet::IpNet, ) -> internal::shared::NetworkInterface { + // TODO-completeness: Handle IP Subnets of either version. + // https://github.com/oxidecomputer/omicron/issues/9246. + assert!( + matches!(subnet, oxnet::IpNet::V4(_)), + "Only IPv4 VPC Subnets are currently supported" + ); internal::shared::NetworkInterface { id: self.id(), kind: match self.kind { @@ -96,7 +127,9 @@ impl NetworkInterface { } }, name: self.name().clone(), - ip: self.ip.ip(), + // TODO-completeness: Handle one or both IP addresses when + // addressing https://github.com/oxidecomputer/omicron/issues/9246. + ip: self.ipv4.expect("only IPv4 interfaces are supported").into(), mac: self.mac.into(), subnet, vni: external::Vni::try_from(0).unwrap(), @@ -117,18 +150,16 @@ impl NetworkInterface { pub struct InstanceNetworkInterface { #[diesel(embed)] pub identity: InstanceNetworkInterfaceIdentity, - pub instance_id: Uuid, pub vpc_id: Uuid, pub subnet_id: Uuid, - pub mac: MacAddr, - pub ip: IpNetwork, - + // NOTE: At least one of the below will be non-None. + pub ipv4: Option, + pub ipv6: Option, pub slot: SqlU8, #[diesel(column_name = is_primary)] pub primary: bool, - pub transit_ips: Vec, } @@ -142,14 +173,13 @@ pub struct InstanceNetworkInterface { pub struct ServiceNetworkInterface { #[diesel(embed)] pub identity: ServiceNetworkInterfaceIdentity, - pub service_id: Uuid, pub vpc_id: Uuid, pub subnet_id: Uuid, - pub mac: MacAddr, - pub ip: IpNetwork, - + // NOTE: At least one of the below will be non-None. + pub ipv4: Option, + pub ipv6: Option, pub slot: SqlU8, #[diesel(column_name = is_primary)] pub primary: bool, @@ -174,35 +204,37 @@ impl ServiceNetworkInterface { } } +// TODO-remove: Remove this when we support dual-stack service NICs. See +// https://github.com/oxidecomputer/omicron/issues/9246. #[derive(Debug, thiserror::Error)] #[error( - "Service NIC {nic_id} has a range of IPs ({ip}); only a single IP is supported" + "Service NIC {nic_id} has an IPv6 address ({ip}); \ + only a single IPv4 address is supported" )] -pub struct ServiceNicNotSingleIpError { +pub struct ServiceNicNotIpv4OnlyError { pub nic_id: Uuid, - pub ip: ipnetwork::IpNetwork, + pub ip: std::net::Ipv6Addr, } impl TryFrom<&'_ ServiceNetworkInterface> for nexus_types::deployment::OmicronZoneNic { - type Error = ServiceNicNotSingleIpError; + type Error = ServiceNicNotIpv4OnlyError; fn try_from(nic: &ServiceNetworkInterface) -> Result { - let size = match nic.ip.size() { - NetworkSize::V4(n) => u128::from(n), - NetworkSize::V6(n) => n, - }; - if size != 1 { - return Err(ServiceNicNotSingleIpError { + if let Some(ipv6) = nic.ipv6 { + return Err(ServiceNicNotIpv4OnlyError { nic_id: nic.id(), - ip: nic.ip, + ip: *ipv6, }); } + let Some(ip) = nic.ipv4 else { + unreachable!("must be single-stack IPv4"); + }; Ok(Self { id: VnicUuid::from_untyped_uuid(nic.id()), mac: *nic.mac, - ip: nic.ip.ip(), + ip: ip.into(), slot: *nic.slot, primary: nic.primary, }) @@ -229,7 +261,8 @@ impl NetworkInterface { vpc_id: self.vpc_id, subnet_id: self.subnet_id, mac: self.mac, - ip: self.ip, + ipv4: self.ipv4, + ipv6: self.ipv6, slot: self.slot, primary: self.primary, transit_ips: self.transit_ips, @@ -255,7 +288,8 @@ impl NetworkInterface { vpc_id: self.vpc_id, subnet_id: self.subnet_id, mac: self.mac, - ip: self.ip, + ipv4: self.ipv4, + ipv6: self.ipv6, slot: self.slot, primary: self.primary, } @@ -278,7 +312,8 @@ impl From for NetworkInterface { vpc_id: iface.vpc_id, subnet_id: iface.subnet_id, mac: iface.mac, - ip: iface.ip, + ipv4: iface.ipv4, + ipv6: iface.ipv6, slot: iface.slot, primary: iface.primary, transit_ips: iface.transit_ips, @@ -302,7 +337,8 @@ impl From for NetworkInterface { vpc_id: iface.vpc_id, subnet_id: iface.subnet_id, mac: iface.mac, - ip: iface.ip, + ipv4: iface.ipv4, + ipv6: iface.ipv6, slot: iface.slot, primary: iface.primary, transit_ips: vec![], @@ -338,6 +374,13 @@ impl IncompleteNetworkInterface { transit_ips: Vec, ) -> Result { if let Some(ip) = ip { + // TODO-completeness: + // https://github.com/oxidecomputer/omicron/issues/9244. + if ip.is_ipv6() { + return Err(Error::invalid_request( + "IPv6 addresses are not yet supported", + )); + } subnet.check_requestable_addr(ip)?; }; if let Some(mac) = mac { @@ -465,12 +508,15 @@ pub struct NetworkInterfaceUpdate { impl From for external::InstanceNetworkInterface { fn from(iface: InstanceNetworkInterface) -> Self { + // TODO-completeness: Support dual-stack in the public API, see + // https://github.com/oxidecomputer/omicron/issues/9248. + let ip = iface.ipv4.expect("only IPv4 addresses").into(); Self { identity: iface.identity(), instance_id: iface.instance_id, vpc_id: iface.vpc_id, subnet_id: iface.subnet_id, - ip: iface.ip.ip(), + ip, mac: *iface.mac, primary: iface.primary, transit_ips: iface diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index ffc6a7a376b..0d3a6761dec 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock}; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: Version = Version::new(199, 0, 0); +pub const SCHEMA_VERSION: Version = Version::new(200, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -28,6 +28,7 @@ static KNOWN_VERSIONS: LazyLock> = LazyLock::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(200, "dual-stack-network-interfaces"), KnownVersion::new(199, "multicast-pool-support"), KnownVersion::new(198, "add-ip-pool-reservation-type-column"), KnownVersion::new(197, "scim-users-and-groups"), diff --git a/nexus/db-model/src/v2p_mapping.rs b/nexus/db-model/src/v2p_mapping.rs index b22891845c4..699fd08b08e 100644 --- a/nexus/db-model/src/v2p_mapping.rs +++ b/nexus/db-model/src/v2p_mapping.rs @@ -1,7 +1,14 @@ -use crate::{Ipv6Addr, MacAddr, Vni}; -use ipnetwork::IpNetwork; +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use crate::Ipv4Addr; +use crate::Ipv6Addr; +use crate::MacAddr; +use crate::Vni; use omicron_uuid_kinds::SledUuid; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; +use serde::Serialize; use uuid::Uuid; /// This is not backed by an actual database view, @@ -15,5 +22,7 @@ pub struct V2PMappingView { pub sled_ip: Ipv6Addr, pub vni: Vni, pub mac: MacAddr, - pub ip: IpNetwork, + // NOTE: At least one of the below will be non-None. + pub ipv4: Option, + pub ipv6: Option, } diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index 82d1d59f984..ed850d4de25 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -4347,7 +4347,7 @@ mod tests { id: *zone_id.as_untyped_uuid(), }, name: Name::from_str("mynic").unwrap(), - ip: "fd77:e9d2:9cd9:2::8".parse().unwrap(), + ip: "172.30.2.6".parse().unwrap(), mac: MacAddr::random_system(), subnet: IpNet::host_net(IpAddr::V6( Ipv6Addr::LOCALHOST, diff --git a/nexus/db-queries/src/db/datastore/deployment/external_networking.rs b/nexus/db-queries/src/db/datastore/deployment/external_networking.rs index b21ce19ad99..5de8818ab46 100644 --- a/nexus/db-queries/src/db/datastore/deployment/external_networking.rs +++ b/nexus/db-queries/src/db/datastore/deployment/external_networking.rs @@ -249,7 +249,20 @@ impl DataStore { // because that would require an extra DB lookup. We'll assume if // these main properties are correct, the subnet is too. for allocated_nic in &allocated_nics { - if allocated_nic.ip.ip() == nic.ip + // TODO-completeness: Need support for dual-stack internal + // network interfaces. See + // https://github.com/oxidecomputer/omicron/issues/9246. + // + // This should not be possible to hit until we actually allow + // creating a NIC with a VPC-private IP address. + let Some(ipv4) = allocated_nic.ipv4 else { + return Err(Error::internal_error(&format!( + "Allocated NICs should be single-stack IPv4, but \ + NIC with id '{}' is missing an IPv4 address", + allocated_nic.identity.id, + ))); + }; + if std::net::IpAddr::from(ipv4) == nic.ip && *allocated_nic.mac == nic.mac && *allocated_nic.slot == nic.slot && allocated_nic.primary == nic.primary @@ -783,7 +796,13 @@ mod tests { assert_eq!(db_nexus_nics[0].vpc_id, NEXUS_VPC_SUBNET.vpc_id); assert_eq!(db_nexus_nics[0].subnet_id, NEXUS_VPC_SUBNET.id()); assert_eq!(*db_nexus_nics[0].mac, self.nexus_nic.mac); - assert_eq!(db_nexus_nics[0].ip, self.nexus_nic.ip.into()); + // TODO-completeness: Handle the `nexus_nic` being dual-stack as + // well. See https://github.com/oxidecomputer/omicron/issues/9246. + assert_eq!( + db_nexus_nics[0].ipv4.map(IpAddr::from), + Some(self.nexus_nic.ip) + ); + assert!(db_nexus_nics[0].ipv6.is_none()); assert_eq!(*db_nexus_nics[0].slot, self.nexus_nic.slot); assert_eq!(db_nexus_nics[0].primary, self.nexus_nic.primary); @@ -803,7 +822,13 @@ mod tests { assert_eq!(db_dns_nics[0].vpc_id, DNS_VPC_SUBNET.vpc_id); assert_eq!(db_dns_nics[0].subnet_id, DNS_VPC_SUBNET.id()); assert_eq!(*db_dns_nics[0].mac, self.dns_nic.mac); - assert_eq!(db_dns_nics[0].ip, self.dns_nic.ip.into()); + // TODO-completeness: Handle the `nexus_nic` being dual-stack as + // well. See https://github.com/oxidecomputer/omicron/issues/9246. + assert_eq!( + db_nexus_nics[0].ipv4.map(IpAddr::from), + Some(self.nexus_nic.ip) + ); + assert!(db_nexus_nics[0].ipv6.is_none()); assert_eq!(*db_dns_nics[0].slot, self.dns_nic.slot); assert_eq!(db_dns_nics[0].primary, self.dns_nic.primary); @@ -823,7 +848,13 @@ mod tests { assert_eq!(db_ntp_nics[0].vpc_id, NTP_VPC_SUBNET.vpc_id); assert_eq!(db_ntp_nics[0].subnet_id, NTP_VPC_SUBNET.id()); assert_eq!(*db_ntp_nics[0].mac, self.ntp_nic.mac); - assert_eq!(db_ntp_nics[0].ip, self.ntp_nic.ip.into()); + // TODO-completeness: Handle the `nexus_nic` being dual-stack as + // well. See https://github.com/oxidecomputer/omicron/issues/9246. + assert_eq!( + db_nexus_nics[0].ipv4.map(IpAddr::from), + Some(self.nexus_nic.ip) + ); + assert!(db_nexus_nics[0].ipv6.is_none()); assert_eq!(*db_ntp_nics[0].slot, self.ntp_nic.slot); assert_eq!(db_ntp_nics[0].primary, self.ntp_nic.primary); } diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index 6c46fd4074f..29d6fff36bb 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -32,6 +32,8 @@ use nexus_db_errors::OptionalError; use nexus_db_errors::public_error_from_diesel; use nexus_db_lookup::DbConnection; use nexus_db_model::IpVersion; +use nexus_db_model::Ipv4Addr; +use nexus_db_model::Ipv6Addr; use nexus_db_model::ServiceNetworkInterface; use nexus_types::identity::Resource; use omicron_common::api::external::DataPageParams; @@ -48,31 +50,62 @@ use uuid::Uuid; /// OPTE requires information that's currently split across the network /// interface and VPC subnet tables. -#[derive(Debug, diesel::Queryable)] +#[derive(Debug, diesel::Queryable, diesel::Selectable)] struct NicInfo { + #[diesel(select_expression = nexus_db_schema::schema::network_interface::id)] id: Uuid, + #[diesel(select_expression = nexus_db_schema::schema::network_interface::parent_id)] parent_id: Uuid, + #[diesel(select_expression = nexus_db_schema::schema::network_interface::kind)] kind: NetworkInterfaceKind, + #[diesel(select_expression = nexus_db_schema::schema::network_interface::name)] name: db::model::Name, - ip: ipnetwork::IpNetwork, + #[diesel(select_expression = nexus_db_schema::schema::network_interface::ip)] + ipv4: Option, + #[diesel(select_expression = nexus_db_schema::schema::network_interface::ipv6)] + _ipv6: Option, + #[diesel(select_expression = nexus_db_schema::schema::network_interface::mac)] mac: db::model::MacAddr, + #[diesel(select_expression = nexus_db_schema::schema::vpc_subnet::ipv4_block)] ipv4_block: db::model::Ipv4Net, - ipv6_block: db::model::Ipv6Net, + #[diesel(select_expression = nexus_db_schema::schema::vpc_subnet::ipv6_block)] + _ipv6_block: db::model::Ipv6Net, + #[diesel(select_expression = nexus_db_schema::schema::vpc::vni)] vni: db::model::Vni, + #[diesel(select_expression = nexus_db_schema::schema::network_interface::is_primary)] primary: bool, + #[diesel(select_expression = nexus_db_schema::schema::network_interface::slot)] slot: i16, + #[diesel(select_expression = nexus_db_schema::schema::network_interface::transit_ips)] transit_ips: Vec, } -impl From for omicron_common::api::internal::shared::NetworkInterface { - fn from( +impl TryFrom + for omicron_common::api::internal::shared::NetworkInterface +{ + type Error = Error; + + fn try_from( nic: NicInfo, - ) -> omicron_common::api::internal::shared::NetworkInterface { - let ip_subnet = if nic.ip.is_ipv4() { - oxnet::IpNet::V4(nic.ipv4_block.0) - } else { - oxnet::IpNet::V6(nic.ipv6_block.0) + ) -> Result< + omicron_common::api::internal::shared::NetworkInterface, + Self::Error, + > { + // TODO-completeness: Support IPv6 and dual-stack NICs in the Nexus <-> + // sled-agent API. That includes the IPs themselves and the VPC Subnets. + // + // See https://github.com/oxidecomputer/omicron/issues/9246. + // + // This whole method can become the infallible `From` again when that's + // resolved. + let Some(ipv4) = nic.ipv4 else { + return Err(Error::internal_error(&format!( + "Found internal NIC without an IPv4 address: \ + nic_id=\"{}\", parent_id=\"{}\"", + nic.id, nic.parent_id, + ))); }; + let ip_subnet = oxnet::IpNet::V4(nic.ipv4_block.0); let kind = match nic.kind { NetworkInterfaceKind::Instance => { omicron_common::api::internal::shared::NetworkInterfaceKind::Instance{ id: nic.parent_id } @@ -84,18 +117,18 @@ impl From for omicron_common::api::internal::shared::NetworkInterface { omicron_common::api::internal::shared::NetworkInterfaceKind::Probe{ id: nic.parent_id } } }; - omicron_common::api::internal::shared::NetworkInterface { + Ok(omicron_common::api::internal::shared::NetworkInterface { id: nic.id, kind, name: nic.name.into(), - ip: nic.ip.ip(), + ip: ipv4.into(), mac: nic.mac.0, subnet: ip_subnet, vni: nic.vni.0, primary: nic.primary, slot: u8::try_from(nic.slot).unwrap(), transit_ips: nic.transit_ips.iter().map(|v| (*v).into()).collect(), - } + }) } } @@ -507,32 +540,14 @@ impl DataStore { ) .inner_join(vpc::table.on(vpc_subnet::vpc_id.eq(vpc::id))) .order_by(network_interface::slot) - // TODO-cleanup: Having to specify each column again is less than - // ideal, but we can't derive `Selectable` since this is the result - // of a JOIN and not from a single table. DRY this out if possible. - .select(( - network_interface::id, - network_interface::parent_id, - network_interface::kind, - network_interface::name, - network_interface::ip, - network_interface::mac, - vpc_subnet::ipv4_block, - vpc_subnet::ipv6_block, - vpc::vni, - network_interface::is_primary, - network_interface::slot, - network_interface::transit_ips, - )) - .get_results_async::( - &*self.pool_connection_authorized(opctx).await?, - ) + .select(NicInfo::as_select()) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; - Ok(rows + rows .into_iter() - .map(omicron_common::api::internal::shared::NetworkInterface::from) - .collect()) + .map(omicron_common::api::internal::shared::NetworkInterface::try_from) + .collect::>() } /// Return the information about an instance's network interfaces required diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index ab4ce8ddb44..68a0e33488e 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -2152,8 +2152,27 @@ mod test { ..Default::default() }, ) - .await - .expect("Failed to initialize rack"); + .await; + + // IPv6 addresses aren't fully supported right now. See + // https://github.com/oxidecomputer/omicron/issues/1716. When that is + // fully-addressed, this will start to fail and we can remove this + // block to restore the previous test coverage. + let Err(Error::InvalidRequest { message }) = &rack else { + panic!( + "Expected an error initializing a rack with an IPv6 address, \ + until they are fully-supported. Found {rack:#?}" + ); + }; + assert_eq!( + message.external_message(), + "IPv6 addresses are not yet supported" + ); + let Ok(rack) = rack else { + db.terminate().await; + logctx.cleanup_successful(); + return; + }; assert_eq!(rack.id(), rack_id()); assert!(rack.initialized); diff --git a/nexus/db-queries/src/db/datastore/v2p_mapping.rs b/nexus/db-queries/src/db/datastore/v2p_mapping.rs index 45e4a359d6f..d413085efb0 100644 --- a/nexus/db-queries/src/db/datastore/v2p_mapping.rs +++ b/nexus/db-queries/src/db/datastore/v2p_mapping.rs @@ -86,7 +86,8 @@ impl DataStore { sled_ip: sled.ip, vni: vpc.vni, mac: nic.mac, - ip: nic.ip, + ipv4: nic.ipv4, + ipv6: nic.ipv6, } }) .collect(); @@ -138,7 +139,8 @@ impl DataStore { sled_ip: sled.ip, vni: vpc.vni, mac: nic.mac, - ip: nic.ip, + ipv4: nic.ipv4, + ipv6: nic.ipv6, } }) .collect(); diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 1e07e37bee7..bb2ff53ccfd 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -2864,14 +2864,14 @@ impl DataStore { .unwrap_or_default(), (RouteTarget::Instance(n), _) => instances .get(&n) - .map(|i| match i.1.ip { - // TODO: update for dual-stack v4/6. - ip @ IpNetwork::V4(_) => { - (Some(RouterTarget::Ip(ip.ip())), None) - } - ip @ IpNetwork::V6(_) => { - (None, Some(RouterTarget::Ip(ip.ip()))) - } + .map(|(_inst, iface)| { + let v4_target = iface + .ipv4 + .map(|ipv4| RouterTarget::Ip(ipv4.into())); + let v6_target = iface + .ipv6 + .map(|ipv6| RouterTarget::Ip(ipv6.into())); + (v4_target, v6_target) }) .unwrap_or_default(), (RouteTarget::Drop, _) => { @@ -2997,6 +2997,8 @@ mod tests { use oxnet::IpNet; use oxnet::Ipv4Net; use slog::info; + use std::net::Ipv4Addr; + use std::net::Ipv6Addr; // Test that we detect the right error condition and return None when we // fail to insert a VPC due to VNI exhaustion. @@ -4048,7 +4050,10 @@ mod tests { assert!(routes.iter().any(|x| (x.dest == "192.168.0.0/16".parse::().unwrap()) && match x.target { - RouterTarget::Ip(ip) => ip == nic.ip.ip(), + RouterTarget::Ip(IpAddr::V4(ipv4)) => + ipv4 == Ipv4Addr::from(nic.ipv4.unwrap()), + RouterTarget::Ip(IpAddr::V6(ipv6)) => + ipv6 == Ipv6Addr::from(nic.ipv6.unwrap()), _ => false, })); diff --git a/nexus/db-queries/src/db/queries/network_interface.rs b/nexus/db-queries/src/db/queries/network_interface.rs index 761d07b5b3f..ae8a059c0ed 100644 --- a/nexus/db-queries/src/db/queries/network_interface.rs +++ b/nexus/db-queries/src/db/queries/network_interface.rs @@ -259,15 +259,19 @@ fn decode_database_error( r#"incorrect UUID length: multiple-vpcs"#, ); - // Error message generated when we attempt to insert NULL in the `ip` - // column, which only happens when we run out of IPs in the subnet. - const IP_EXHAUSTION_ERROR_MESSAGE: &str = - r#"null value in column "ip" violates not-null constraint"#; + // The name of the constraint violated when we attempt to insert NULL for + // one of the IP address columns, either `ipv4` or `ipv6`. This only happens + // when we run out of addresses in the subnet. + const IP_EXHAUSTION_CONSTRAINT: &str = r#"at_least_one_ip_address"#; // The name of the index whose uniqueness is violated if we try to assign an - // IP that is already allocated to another interface in the same subnet. - const IP_NOT_AVAILABLE_CONSTRAINT: &str = - "network_interface_subnet_id_ip_key"; + // IPv4 that is already allocated to another interface in the same subnet. + const IPV4_NOT_AVAILABLE_CONSTRAINT: &str = + "network_interface_subnet_id_ipv4_key"; + + // TODO-completeness: Add a similar constraint name and check below when we + // support inserting VPC-private IPv6 addresses. See + // https://github.com/oxidecomputer/omicron/issues/9245. // The name of the index whose uniqueness is violated if we try to assign a // MAC that is already allocated to another interface in the same VPC. @@ -321,12 +325,11 @@ fn decode_database_error( match err { // If the address allocation subquery fails, we'll attempt to insert - // NULL for the `ip` column. This checks that the non-NULL constraint on - // that colum has been violated. - DieselError::DatabaseError( - DatabaseErrorKind::NotNullViolation, - info, - ) if info.message() == IP_EXHAUSTION_ERROR_MESSAGE => { + // NULL for one of the IP columns. This checks if the CHECK constraint + // on the table has been violated. + DieselError::DatabaseError(DatabaseErrorKind::CheckViolation, info) + if info.constraint_name() == Some(IP_EXHAUSTION_CONSTRAINT) => + { InsertError::NoAvailableIpAddresses { name: interface.subnet.identity.name.to_string(), id: interface.subnet.identity.id, @@ -397,7 +400,7 @@ fn decode_database_error( ) => match info.constraint_name() { // Constraint violated if a user-requested IP address has // already been assigned within the same VPC Subnet. - Some(constraint) if constraint == IP_NOT_AVAILABLE_CONSTRAINT => { + Some(constraint) if constraint == IPV4_NOT_AVAILABLE_CONSTRAINT => { let ip = interface .ip .unwrap_or_else(|| std::net::Ipv4Addr::UNSPECIFIED.into()); @@ -1949,7 +1952,7 @@ mod tests { let new_runtime = model::InstanceRuntimeState { nexus_state: state, propolis_id, - gen: instance.runtime_state.gen.next().into(), + r#gen: instance.runtime_state.gen.next().into(), ..instance.runtime_state.clone() }; let res = db_datastore @@ -2249,10 +2252,14 @@ mod tests { .expect("Failed to insert interface with known-good IP address"); assert_interfaces_eq(&interface, &inserted_interface.clone().into()); assert_eq!( - inserted_interface.ip.ip(), + IpAddr::from(inserted_interface.ipv4.expect("an IPv4 address")), requested_ip, "The requested IP address should be available when no interfaces exist in the table" ); + assert!( + inserted_interface.ipv6.is_none(), + "Should not have an IPv6 address" + ); context.success().await; } @@ -2323,11 +2330,17 @@ mod tests { &interface, &inserted_interface.clone().into(), ); - let actual_address = inserted_interface.ip.ip(); + let actual_address = Ipv4Addr::from( + inserted_interface.ipv4.expect("an IPv4 address"), + ); assert_eq!( actual_address, expected_address, "Failed to auto-assign correct sequential address to interface" ); + assert!( + inserted_interface.ipv6.is_none(), + "Should not have an IPv6 address" + ); } context.success().await; } @@ -2364,6 +2377,7 @@ mod tests { // Inserting an interface with the same IP should fail, even if all // other parameters are valid. + let ip = inserted_interface.ipv4.expect("an IPv4 address"); let interface = IncompleteNetworkInterface::new_instance( Uuid::new_v4(), new_instance_id, @@ -2372,7 +2386,7 @@ mod tests { name: "interface-c".parse().unwrap(), description: String::from("description"), }, - Some(inserted_interface.ip.ip()), + Some(ip.into()), vec![], ) .unwrap(); @@ -2380,10 +2394,12 @@ mod tests { .datastore() .instance_create_network_interface_raw(context.opctx(), interface) .await; - assert!( - matches!(result, Err(InsertError::IpAddressNotAvailable(_))), - "Requesting an interface with an existing IP should fail" - ); + let Err(InsertError::IpAddressNotAvailable(_)) = result else { + panic!( + "Requesting an interface with an existing IP should fail, found {:?}", + result, + ); + }; context.success().await; } @@ -3047,7 +3063,7 @@ mod tests { } // Delete the NIC on the first instance. - let original_ip = instances[0].1.ip.ip(); + let original_ip = instances[0].1.ipv4.expect("an IPv4 address"); context.delete_instance_nics(instances[0].0.id()).await; // And recreate it, ensuring we get the same IP address again. @@ -3069,9 +3085,9 @@ mod tests { .await .expect("Failed to insert interface"); instances[0].1 = intf; + let new_ip = instances[0].1.ipv4.expect("an IPv4 address"); assert_eq!( - instances[0].1.ip.ip(), - original_ip, + new_ip, original_ip, "Should have recreated the first available IP address again" ); @@ -3101,8 +3117,7 @@ mod tests { .await .expect("Failed to insert interface"); assert_eq!( - intf.ip.ip(), - instances[1].1.ip.ip(), + intf.ipv4, instances[1].1.ipv4, "Should have used the second address", ); @@ -3178,9 +3193,10 @@ mod tests { panic!("Should have been able to get the {NTH}-1-th address") }) ), - interface2.ip.ip(), + IpAddr::from(interface2.ipv4.expect("an IPv4 address")), "Should have allocated 1 less than the smallest existing address" ); + assert!(interface2.ipv6.is_none(), "Should not have an IPv6 address"); context.success().await; } diff --git a/nexus/db-schema/src/schema.rs b/nexus/db-schema/src/schema.rs index 654a11b8f3e..747323c2e9a 100644 --- a/nexus/db-schema/src/schema.rs +++ b/nexus/db-schema/src/schema.rs @@ -575,7 +575,11 @@ table! { vpc_id -> Uuid, subnet_id -> Uuid, mac -> Int8, - ip -> Inet, + // NOTE: This is the IPv4 address, despite the name. We kept the + // original name of `ip` because renaming columns is not idempotent in + // CRDB as of today. + ip -> Nullable, + ipv6 -> Nullable, slot -> Int2, is_primary -> Bool, transit_ips -> Array, @@ -594,7 +598,8 @@ table! { vpc_id -> Uuid, subnet_id -> Uuid, mac -> Int8, - ip -> Inet, + ipv4 -> Nullable, + ipv6 -> Nullable, slot -> Int2, is_primary -> Bool, transit_ips -> Array, @@ -614,7 +619,8 @@ table! { vpc_id -> Uuid, subnet_id -> Uuid, mac -> Int8, - ip -> Inet, + ipv4 -> Nullable, + ipv6 -> Nullable, slot -> Int2, is_primary -> Bool, } diff --git a/nexus/src/app/background/tasks/v2p_mappings.rs b/nexus/src/app/background/tasks/v2p_mappings.rs index c095771c610..4c396140d60 100644 --- a/nexus/src/app/background/tasks/v2p_mappings.rs +++ b/nexus/src/app/background/tasks/v2p_mappings.rs @@ -1,4 +1,8 @@ -use std::{collections::HashSet, sync::Arc}; +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::{collections::HashSet, net::IpAddr, sync::Arc}; use futures::FutureExt; use futures::future::BoxFuture; @@ -68,13 +72,25 @@ impl BackgroundTask for V2PManager { // create a set of updates from the v2p mappings let desired_v2p: HashSet<_> = v2p_mappings .into_iter() - .map(|mapping| { - VirtualNetworkInterfaceHost { - virtual_ip: mapping.ip.ip(), + .filter_map(|mapping| { + // TODO-completeness: Support dual-stack in the + // `VirtualNetworkInterfaceHost` type. See + // https://github.com/oxidecomputer/omicron/issues/9246. + let Some(virtual_ip) = mapping.ipv4.map(IpAddr::from) else { + error!( + &log, + "No IPv4 address in V2P mapping"; + "nic_id" => %mapping.nic_id, + "sled_id" => %mapping.sled_id, + ); + return None; + }; + Some(VirtualNetworkInterfaceHost { + virtual_ip, virtual_mac: *mapping.mac, physical_host_ip: *mapping.sled_ip, vni: mapping.vni.0, - } + }) }) .collect(); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index c927c560c17..8c8afbf0754 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -1850,18 +1850,25 @@ CREATE TABLE IF NOT EXISTS omicron.public.network_interface ( */ mac INT8 NOT NULL, - /* The private VPC IP address of the interface. */ - ip INET NOT NULL, + /* The private VPC IPv4 address of the interface. + * + * At least one of the IPv4 and IPv6 addresses must be specified. + * + * NOTE: Despite the name, this is in fact the IPv4 address. We've kept the + * original name `ip` since renaming columns idempotently is difficult in + * CRDB right now. + */ + ip INET, /* * Limited to 8 NICs per instance. This value must be kept in sync with - * `crate::nexus::MAX_NICS_PER_INSTANCE`. + * `nexus_db_model::MAX_NICS_PER_INSTANCE`. */ slot INT2 NOT NULL CHECK (slot >= 0 AND slot < 8), /* True if this interface is the primary interface. * - * The primary interface appears in DNS and its address is used for external + * The primary interface appears in DNS and its addresses are used for external * connectivity. */ is_primary BOOL NOT NULL, @@ -1871,7 +1878,20 @@ CREATE TABLE IF NOT EXISTS omicron.public.network_interface ( * *allowed* to send/receive traffic on, in addition to its * assigned address. */ - transit_ips INET[] NOT NULL DEFAULT ARRAY[] + transit_ips INET[] NOT NULL DEFAULT ARRAY[], + + /* The private VPC IPv6 address of the interface. + * + * At least one of the IPv4 and IPv6 addresses must be specified. + */ + ipv6 INET, + + /* Constraint ensuring we have at least one IP address from either family. + * Both may be specified. + */ + CONSTRAINT at_least_one_ip_address CHECK ( + ip IS NOT NULL OR ipv6 IS NOT NULL + ) ); CREATE INDEX IF NOT EXISTS instance_network_interface_mac @@ -1890,7 +1910,8 @@ SELECT vpc_id, subnet_id, mac, - ip, + ip AS ipv4, + ipv6, slot, is_primary, transit_ips @@ -1912,7 +1933,8 @@ SELECT vpc_id, subnet_id, mac, - ip, + ip AS ipv4, + ipv6, slot, is_primary FROM @@ -1928,12 +1950,17 @@ WHERE * as moving IPs between NICs on different instances, etc. */ -/* Ensure we do not assign the same address twice within a subnet */ -CREATE UNIQUE INDEX IF NOT EXISTS network_interface_subnet_id_ip_key ON omicron.public.network_interface ( +/* Ensure we do not assign the same addresses twice within a subnet */ +CREATE UNIQUE INDEX IF NOT EXISTS network_interface_subnet_id_ipv4_key ON omicron.public.network_interface ( subnet_id, ip ) WHERE - time_deleted IS NULL; + time_deleted IS NULL AND ip IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS network_interface_subnet_id_ipv6_key ON omicron.public.network_interface ( + subnet_id, + ipv6 +) WHERE + time_deleted IS NULL AND ipv6 IS NOT NULL; /* Ensure we do not assign the same MAC twice within a VPC * See RFD174's discussion on the scope of virtual MACs @@ -1973,6 +2000,22 @@ CREATE UNIQUE INDEX IF NOT EXISTS network_interface_parent_id_slot_key ON omicro WHERE time_deleted IS NULL; +/* + * Index used to look up NIC details by its parent ID. + */ +CREATE INDEX IF NOT EXISTS network_interface_by_parent +ON omicron.public.network_interface (parent_id) +STORING (name, kind, vpc_id, subnet_id, mac, ip, ipv6, slot); + +/* + * Index used to select details needed to build the + * virtual-to-physical mappings quickly. + */ +CREATE INDEX IF NOT EXISTS v2p_mapping_details +ON omicron.public.network_interface ( + time_deleted, kind, subnet_id, vpc_id, parent_id +) STORING (mac, ip, ipv6); + CREATE TYPE IF NOT EXISTS omicron.public.vpc_firewall_rule_status AS ENUM ( 'disabled', 'enabled' @@ -5662,21 +5705,12 @@ ON omicron.public.switch_port (port_settings_id, port_name) STORING (switch_loca CREATE INDEX IF NOT EXISTS switch_port_name ON omicron.public.switch_port (port_name); -CREATE INDEX IF NOT EXISTS network_interface_by_parent -ON omicron.public.network_interface (parent_id) -STORING (name, kind, vpc_id, subnet_id, mac, ip, slot); - CREATE INDEX IF NOT EXISTS sled_by_policy_and_state ON omicron.public.sled (sled_policy, sled_state, id) STORING (ip); CREATE INDEX IF NOT EXISTS active_vmm ON omicron.public.vmm (time_deleted, sled_id, instance_id); -CREATE INDEX IF NOT EXISTS v2p_mapping_details -ON omicron.public.network_interface ( - time_deleted, kind, subnet_id, vpc_id, parent_id -) STORING (mac, ip); - CREATE INDEX IF NOT EXISTS sled_by_policy ON omicron.public.sled (sled_policy) STORING (ip, sled_state); @@ -6792,7 +6826,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '199.0.0', NULL) + (TRUE, NOW(), NOW(), '200.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/dual-stack-network-interfaces/up01.sql b/schema/crdb/dual-stack-network-interfaces/up01.sql new file mode 100644 index 00000000000..cba24cdc502 --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up01.sql @@ -0,0 +1,2 @@ +-- Drop the index which will depend on the `ipv6` column we're about to add. +DROP INDEX IF EXISTS network_interface@network_interface_by_parent; diff --git a/schema/crdb/dual-stack-network-interfaces/up02.sql b/schema/crdb/dual-stack-network-interfaces/up02.sql new file mode 100644 index 00000000000..f78022378a6 --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up02.sql @@ -0,0 +1,2 @@ +-- Drop the index which will depend on the `ipv6` column we're about to add. +DROP INDEX IF EXISTS network_interface@v2p_mapping_details; diff --git a/schema/crdb/dual-stack-network-interfaces/up03.sql b/schema/crdb/dual-stack-network-interfaces/up03.sql new file mode 100644 index 00000000000..f675c9b9f35 --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up03.sql @@ -0,0 +1,2 @@ +-- Drop the view which will depend on the `ipv6` column we're about to add. +DROP VIEW IF EXISTS omicron.public.instance_network_interface; diff --git a/schema/crdb/dual-stack-network-interfaces/up04.sql b/schema/crdb/dual-stack-network-interfaces/up04.sql new file mode 100644 index 00000000000..8b440399034 --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up04.sql @@ -0,0 +1,2 @@ +-- Drop the view which will depend on the `ipv6` column we're about to add. +DROP VIEW IF EXISTS omicron.public.service_network_interface; diff --git a/schema/crdb/dual-stack-network-interfaces/up05.sql b/schema/crdb/dual-stack-network-interfaces/up05.sql new file mode 100644 index 00000000000..ea4353133c7 --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up05.sql @@ -0,0 +1,2 @@ +-- Drop the view which will depend on the `ipv6` column we're about to add. +DROP INDEX IF EXISTS omicron.public.network_interface_subnet_id_ip_key; diff --git a/schema/crdb/dual-stack-network-interfaces/up06.sql b/schema/crdb/dual-stack-network-interfaces/up06.sql new file mode 100644 index 00000000000..484615ed010 --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up06.sql @@ -0,0 +1,4 @@ +-- Add the `ipv6` column first. +ALTER TABLE IF EXISTS +omicron.public.network_interface +ADD COLUMN IF NOT EXISTS ipv6 INET; diff --git a/schema/crdb/dual-stack-network-interfaces/up07.sql b/schema/crdb/dual-stack-network-interfaces/up07.sql new file mode 100644 index 00000000000..87d6d6bf4c9 --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up07.sql @@ -0,0 +1,5 @@ +-- Drop the non-NULL constraint on the `ip` column, since an interface +-- can be IPv6-only in the future. +ALTER TABLE IF EXISTS +omicron.public.network_interface +ALTER COLUMN ip DROP NOT NULL; diff --git a/schema/crdb/dual-stack-network-interfaces/up08.sql b/schema/crdb/dual-stack-network-interfaces/up08.sql new file mode 100644 index 00000000000..6f2860f10db --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up08.sql @@ -0,0 +1,6 @@ +-- Add constraint that ensures we have at least on IP, from either family. +ALTER TABLE IF EXISTS +omicron.public.network_interface +ADD CONSTRAINT IF NOT EXISTS at_least_one_ip_address CHECK ( + ip IS NOT NULL OR ipv6 IS NOT NULL +); diff --git a/schema/crdb/dual-stack-network-interfaces/up09.sql b/schema/crdb/dual-stack-network-interfaces/up09.sql new file mode 100644 index 00000000000..65d398147d7 --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up09.sql @@ -0,0 +1,21 @@ +-- Recreate the view we dropped earlier to add the ipv6 column +CREATE VIEW IF NOT EXISTS omicron.public.service_network_interface AS +SELECT + id, + name, + description, + time_created, + time_modified, + time_deleted, + parent_id AS service_id, + vpc_id, + subnet_id, + mac, + ip AS ipv4, + ipv6, + slot, + is_primary +FROM + omicron.public.network_interface +WHERE + kind = 'service'; diff --git a/schema/crdb/dual-stack-network-interfaces/up10.sql b/schema/crdb/dual-stack-network-interfaces/up10.sql new file mode 100644 index 00000000000..e4add394e0a --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up10.sql @@ -0,0 +1,22 @@ +-- Recreate the view we dropped earlier to add the `ipv6` column +CREATE VIEW IF NOT EXISTS omicron.public.instance_network_interface AS +SELECT + id, + name, + description, + time_created, + time_modified, + time_deleted, + parent_id AS instance_id, + vpc_id, + subnet_id, + mac, + ip AS ipv4, + ipv6, + slot, + is_primary, + transit_ips +FROM + omicron.public.network_interface +WHERE + kind = 'instance'; diff --git a/schema/crdb/dual-stack-network-interfaces/up11.sql b/schema/crdb/dual-stack-network-interfaces/up11.sql new file mode 100644 index 00000000000..ae881eb57bd --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up11.sql @@ -0,0 +1,9 @@ +-- Recreate the index on ip we dropped earlier to rename the column. +-- NOTE: It has a new name here. +CREATE UNIQUE INDEX IF NOT EXISTS +network_interface_subnet_id_ipv4_key +ON omicron.public.network_interface ( + subnet_id, + ip +) WHERE + time_deleted IS NULL AND ip IS NOT NULL; diff --git a/schema/crdb/dual-stack-network-interfaces/up12.sql b/schema/crdb/dual-stack-network-interfaces/up12.sql new file mode 100644 index 00000000000..e1d452edb6b --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up12.sql @@ -0,0 +1,8 @@ +-- Create the new index on `ipv6`. +CREATE UNIQUE INDEX IF NOT EXISTS +network_interface_subnet_id_ipv6_key +ON omicron.public.network_interface ( + subnet_id, + ipv6 +) WHERE + time_deleted IS NULL AND ipv6 IS NOT NULL; diff --git a/schema/crdb/dual-stack-network-interfaces/up13.sql b/schema/crdb/dual-stack-network-interfaces/up13.sql new file mode 100644 index 00000000000..b9c3bdcf322 --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up13.sql @@ -0,0 +1,5 @@ +-- Recreate the index on NICs by parent that we dropped +-- to add the `ipv6` column to it. +CREATE INDEX IF NOT EXISTS network_interface_by_parent +ON omicron.public.network_interface (parent_id) +STORING (name, kind, vpc_id, subnet_id, mac, ip, ipv6, slot); diff --git a/schema/crdb/dual-stack-network-interfaces/up14.sql b/schema/crdb/dual-stack-network-interfaces/up14.sql new file mode 100644 index 00000000000..d88e1f95c75 --- /dev/null +++ b/schema/crdb/dual-stack-network-interfaces/up14.sql @@ -0,0 +1,6 @@ +-- Recreate the index on NICs for V2P mapping details that +-- we dropped to add the `ipv6` column to it. +CREATE INDEX IF NOT EXISTS v2p_mapping_details +ON omicron.public.network_interface ( + time_deleted, kind, subnet_id, vpc_id, parent_id +) STORING (mac, ip, ipv6);