Specify external DNS IPs in PlanningInput instead of deriving them from the parent blueprint (#9291)

This is built on top of #9250, and is related to both #9238 and #8949:

* Trims down some of the `BlueprintResourceAllocator` internals
* Adds an explicit `ExternalIpPolicy` type; today it holds a couple of
IP pools and a set of external DNS IPs, but it gives us a cleaner place
to integrate all the IP pool work that @bnaecker is doing with
Reconfigurator

Since we don't actually have external DNS IPs in a policy anywhere, this
adds `DataStore::external_dns_external_ips_specified_by_rack_setup()`
(which is implemented as "load the current target blueprint and scan it
for external DNS zones"; i.e., exactly what the builder was doing
before). We should eventually delete this method once it's possible for
an operator to reconfigure the external DNS IPs, but this lets the
planning input build up the policy as though we already had this
information available, more or less. Other than that new method,
_almost_ all the changes in this PR are to tests; the non-test changes
are pretty minimal.

Author: jgallagher

common/src/address.rs

@@ -628,6 +628,16 @@ impl Ipv4Range {
         let end_num = u32::from(self.last);
         end_num - start_num + 1
     }
+
+    /// Returns `true` if `self` has any IPs in common with `other`; false
+    /// otherwise.
+    pub fn overlaps(&self, other: &Ipv4Range) -> bool {
+        // We're disjoint if we either end before other or begin after it; any
+        // other combination means we have some IP(s) in common.
+        let is_disjoint = self.last_address() < other.first_address()
+            || self.first_address() > other.last_address();
+        !is_disjoint
+    }
 }
 
 impl From<Ipv4Addr> for Ipv4Range {
@@ -701,6 +711,16 @@ impl Ipv6Range {
         let end_num = u128::from(self.last);
         end_num - start_num + 1
     }
+
+    /// Returns `true` if `self` has any IPs in common with `other`; false
+    /// otherwise.
+    pub fn overlaps(&self, other: &Ipv6Range) -> bool {
+        // We're disjoint if we either end before other or begin after it; any
+        // other combination means we have some IP(s) in common.
+        let is_disjoint = self.last_address() < other.first_address()
+            || self.first_address() > other.last_address();
+        !is_disjoint
+    }
 }
 
 impl From<Ipv6Addr> for Ipv6Range {

dev-tools/reconfigurator-cli/tests/output/cmds-stdout

@@ -871,7 +871,7 @@ result:
     loaded sleds: 04ef3330-c682-4a08-8def-fcc4bef31bcd, 90c1102a-b9f5-4d88-92a2-60d54a2d98cc, dde1c0e2-b10d-4621-b420-f179f7a7a00a
     loaded collections: 6e066695-94bc-4250-bd63-fd799c166cc1
     loaded blueprints: (none)
-    loaded service IP pool ranges: [V4(Ipv4Range { first: 192.0.2.2, last: 192.0.2.20 })]
+    loaded external IP policy: ExternalIpPolicy { service_pool_ipv4_ranges: [Ipv4Range { first: 192.0.2.2, last: 192.0.2.20 }], service_pool_ipv6_ranges: [], external_dns_ips: {} }
     loaded internal DNS generations: (none)
     loaded external DNS generations: (none)
   config:

dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout

@@ -1038,7 +1038,7 @@ result:
     loaded sleds: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, d81c6a84-79b8-4958-ae41-ea46c9b19763
     loaded collections: f45ba181-4b56-42cc-a762-874d90184a43, eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51, 61f451b3-2121-4ed6-91c7-a550054f6c21
     loaded blueprints: 184f10b3-61cb-41ef-9b93-3489b2bac559, dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21, 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1, 58d5e830-0884-47d8-a7cd-b2b3751adeb4
-    loaded service IP pool ranges: [V4(Ipv4Range { first: 192.0.2.2, last: 192.0.2.20 })]
+    loaded external IP policy: ExternalIpPolicy { service_pool_ipv4_ranges: [Ipv4Range { first: 192.0.2.2, last: 192.0.2.20 }, Ipv4Range { first: 198.51.100.1, last: 198.51.100.30 }], service_pool_ipv6_ranges: [], external_dns_ips: {198.51.100.1, 198.51.100.2, 198.51.100.3} }
     loaded internal DNS generations: (none)
     loaded external DNS generations: (none)
   config:

nexus/db-queries/src/db/datastore/deployment/external_networking.rs

@@ -16,6 +16,8 @@ use nexus_db_model::IpConfig;
 use nexus_db_model::IpPool;
 use nexus_sled_agent_shared::inventory::ZoneKind;
 use nexus_types::deployment::BlueprintZoneConfig;
+use nexus_types::deployment::BlueprintZoneDisposition;
+use nexus_types::deployment::BlueprintZoneType;
 use nexus_types::deployment::OmicronZoneExternalIp;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::IdentityMetadataCreateParams;
@@ -30,8 +32,57 @@ use slog::error;
 use slog::info;
 use slog::warn;
 use slog_error_chain::InlineErrorChain;
+use std::collections::BTreeSet;
+use std::net::IpAddr;
 
 impl DataStore {
+    /// Return the set of external IPs configured for our external DNS servers
+    /// when the rack was set up.
+    ///
+    /// We should have explicit storage for the external IPs on which we run
+    /// external DNS that an operator can update. Today, we do not: whatever
+    /// external DNS IPs are provided at rack setup time are the IPs we use
+    /// forever. (Fixing this is tracked by
+    /// <https://github.com/oxidecomputer/omicron/issues/8255>.)
+    pub async fn external_dns_external_ips_specified_by_rack_setup(
+        &self,
+        opctx: &OpContext,
+    ) -> Result<BTreeSet<IpAddr>, Error> {
+        // We can _implicitly_ determine the set of external DNS IPs provided
+        // during rack setup by examining the current target blueprint and
+        // looking at the IPs of all of its external DNS zones. We _must_
+        // include expunged zones as well as in-service zones: during an update,
+        // we'll create a blueprint that expunges an external DNS zone, waits
+        // for it to go away, then wants to reassign that zone's external IP to
+        // a new external DNS zone. But because we are scanning expunged zones,
+        // we also have to allow for duplicates - this isn't an error and is
+        // expected if we've performed more than one update, at least until we
+        // start pruning old expunged zones out of the blueprint (tracked by
+        // https://github.com/oxidecomputer/omicron/issues/5552).
+        //
+        // Because we can't (yet) change external DNS IPs, we don't have to
+        // worry about the current blueprint changing between when we read it
+        // and when we calculate the set of external DNS IPs: the set will be
+        // identical for all blueprints back to the original one created by RSS.
+        //
+        // We don't really need to load the entire blueprint here, but it's easy
+        // and ideally this code will be deleted in relatively short order.
+        let (_target, blueprint) =
+            self.blueprint_target_get_current_full(opctx).await?;
+
+        let external_dns_ips = blueprint
+            .all_omicron_zones(BlueprintZoneDisposition::any)
+            .filter_map(|(_sled_id, z)| match &z.zone_type {
+                BlueprintZoneType::ExternalDns(external_dns) => {
+                    Some(external_dns.dns_address.addr.ip())
+                }
+                _ => None,
+            })
+            .collect();
+
+        Ok(external_dns_ips)
+    }
+
     pub(super) async fn ensure_zone_external_networking_allocated_on_connection(
         &self,
         conn: &async_bb8_diesel::Connection<DbConnection>,
@@ -461,9 +512,10 @@ mod tests {
     use chrono::Utc;
     use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
     use nexus_db_model::SqlU16;
+    use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder;
     use nexus_sled_agent_shared::inventory::OmicronZoneDataset;
+    use nexus_types::deployment::BlueprintTarget;
     use nexus_types::deployment::BlueprintZoneConfig;
-    use nexus_types::deployment::BlueprintZoneDisposition;
     use nexus_types::deployment::BlueprintZoneImageSource;
     use nexus_types::deployment::BlueprintZoneType;
     use nexus_types::deployment::OmicronZoneExternalFloatingAddr;
@@ -483,10 +535,12 @@ mod tests {
     use omicron_common::api::external::Vni;
     use omicron_common::zpool_name::ZpoolName;
     use omicron_test_utils::dev;
+    use omicron_uuid_kinds::BlueprintUuid;
     use omicron_uuid_kinds::ExternalIpUuid;
+    use omicron_uuid_kinds::ExternalZpoolUuid;
+    use omicron_uuid_kinds::SledUuid;
     use omicron_uuid_kinds::ZpoolUuid;
     use oxnet::IpNet;
-    use std::collections::BTreeSet;
     use std::net::IpAddr;
     use std::net::SocketAddr;
     use uuid::Uuid;
@@ -1292,4 +1346,150 @@ mod tests {
         db.terminate().await;
         logctx.cleanup_successful();
     }
+
+    #[tokio::test]
+    async fn test_external_dns_external_ips_specified_by_rack_setup() {
+        const TEST_NAME: &str =
+            "test_external_dns_external_ips_specified_by_rack_setup";
+
+        // Helper closures to reduce boilerplate below.
+        let make_bp_target = |blueprint_id| BlueprintTarget {
+            target_id: blueprint_id,
+            enabled: false,
+            time_made_target: Utc::now(),
+        };
+        let mut opte_ip_iter = DNS_OPTE_IPV4_SUBNET.addr_iter();
+        let mut mac_iter = MacAddr::iter_system();
+        let mut make_external_dns_zone = |ip, disposition| {
+            let zone_id = OmicronZoneUuid::new_v4();
+            let pool = ZpoolName::External(ExternalZpoolUuid::new_v4());
+            BlueprintZoneConfig {
+                disposition,
+                id: zone_id,
+                filesystem_pool: pool,
+                zone_type: BlueprintZoneType::ExternalDns(
+                    blueprint_zone_type::ExternalDns {
+                        dataset: OmicronZoneDataset { pool_name: pool },
+                        http_address: "[::1]:0".parse().unwrap(),
+                        dns_address: OmicronZoneExternalFloatingAddr {
+                            id: ExternalIpUuid::new_v4(),
+                            addr: SocketAddr::new(ip, 0),
+                        },
+                        nic: NetworkInterface {
+                            id: Uuid::new_v4(),
+                            kind: NetworkInterfaceKind::Service {
+                                id: zone_id.into_untyped_uuid(),
+                            },
+                            name: "test-external-dns".parse().unwrap(),
+                            ip: opte_ip_iter.next().unwrap().into(),
+                            mac: mac_iter.next().unwrap(),
+                            subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET),
+                            vni: Vni::SERVICES_VNI,
+                            primary: true,
+                            slot: 0,
+                            transit_ips: vec![],
+                        },
+                    },
+                ),
+                image_source: BlueprintZoneImageSource::InstallDataset,
+            }
+        };
+
+        // Set up.
+        let logctx = dev::test_setup_log(TEST_NAME);
+        let db = TestDatabase::new_with_datastore(&logctx.log).await;
+        let (opctx, datastore) = (db.opctx(), db.datastore());
+
+        // Create a blueprint with one sled and no zones. Insert it and make it
+        // the target.
+        let sled_id = SledUuid::new_v4();
+        let bp0 = BlueprintBuilder::build_empty_with_sleds(
+            std::iter::once(sled_id),
+            TEST_NAME,
+        );
+        datastore.blueprint_insert(opctx, &bp0).await.expect("inserted bp0");
+        datastore
+            .blueprint_target_set_current(opctx, make_bp_target(bp0.id))
+            .await
+            .expect("made bp0 the target");
+
+        // No external DNS zones => no external DNS IPs.
+        let external_dns_ips = datastore
+            .external_dns_external_ips_specified_by_rack_setup(opctx)
+            .await
+            .expect("got external DNS IPs");
+        assert_eq!(external_dns_ips, BTreeSet::new());
+
+        // Create a blueprint with three in-service external DNS zones. We
+        // should get their IPs back.
+        let expected_ips = ["192.168.1.1", "192.168.1.2", "192.168.1.3"]
+            .into_iter()
+            .map(|ip| ip.parse::<IpAddr>().unwrap())
+            .collect::<BTreeSet<_>>();
+        let mut bp1 = bp0.clone();
+        bp1.id = BlueprintUuid::new_v4();
+        bp1.parent_blueprint_id = Some(bp0.id);
+        for &ip in &expected_ips {
+            bp1.sleds.get_mut(&sled_id).unwrap().zones.insert(
+                make_external_dns_zone(ip, BlueprintZoneDisposition::InService),
+            );
+        }
+
+        // Insert bp1 and make it the target. Confirm we get back the expected
+        // external DNS IPs.
+        datastore.blueprint_insert(opctx, &bp1).await.expect("inserted bp1");
+        datastore
+            .blueprint_target_set_current(opctx, make_bp_target(bp1.id))
+            .await
+            .expect("made bp1 the target");
+        let external_dns_ips = datastore
+            .external_dns_external_ips_specified_by_rack_setup(opctx)
+            .await
+            .expect("got external DNS IPs");
+        assert_eq!(external_dns_ips, expected_ips);
+
+        // Create a third blueprint with multiple expunged external DNS zones
+        // covering a couple additional IPs. Those should also be returned.
+        let extra_ips = ["192.168.1.4", "192.168.1.5"]
+            .into_iter()
+            .map(|ip| ip.parse::<IpAddr>().unwrap())
+            .collect::<BTreeSet<_>>();
+        assert_eq!(expected_ips.intersection(&extra_ips).count(), 0);
+
+        let mut bp2 = bp1.clone();
+        bp2.id = BlueprintUuid::new_v4();
+        bp2.parent_blueprint_id = Some(bp1.id);
+        for &ip in &extra_ips {
+            for i in 0..4 {
+                bp2.sleds.get_mut(&sled_id).unwrap().zones.insert(
+                    make_external_dns_zone(
+                        ip,
+                        BlueprintZoneDisposition::Expunged {
+                            as_of_generation: Generation::new(),
+                            ready_for_cleanup: i % 2 == 0,
+                        },
+                    ),
+                );
+            }
+        }
+
+        // Insert bp1 and make it the target. Confirm we get back the expected
+        // external DNS IPs.
+        datastore.blueprint_insert(opctx, &bp2).await.expect("inserted bp2");
+        datastore
+            .blueprint_target_set_current(opctx, make_bp_target(bp2.id))
+            .await
+            .expect("made bp2 the target");
+        let external_dns_ips = datastore
+            .external_dns_external_ips_specified_by_rack_setup(opctx)
+            .await
+            .expect("got external DNS IPs");
+        let expected_ips =
+            expected_ips.union(&extra_ips).copied().collect::<BTreeSet<_>>();
+        assert_eq!(external_dns_ips, expected_ips);
+
+        // Clean up.
+        db.terminate().await;
+        logctx.cleanup_successful();
+    }
 }