add DNS names for TUF repo depot (#7939)

Author: davepacheco

dev-tools/reconfigurator-cli/src/lib.rs

@@ -34,6 +34,9 @@ use nexus_types::deployment::execution;
 use nexus_types::deployment::execution::blueprint_external_dns_config;
 use nexus_types::deployment::execution::blueprint_internal_dns_config;
 use nexus_types::deployment::{Blueprint, UnstableReconfiguratorState};
+use nexus_types::external_api::views::SledPolicy;
+use nexus_types::external_api::views::SledProvisionPolicy;
+use omicron_common::address::REPO_DEPOT_PORT;
 use omicron_common::api::external::Generation;
 use omicron_common::api::external::Name;
 use omicron_common::policy::NEXUS_REDUNDANCY;
@@ -1082,7 +1085,11 @@ fn make_sleds_by_id(
         .map(|(sled_id, sled_agent_info)| {
             let sled = execution::Sled::new(
                 *sled_id,
+                SledPolicy::InService {
+                    provision_policy: SledProvisionPolicy::Provisionable,
+                },
                 sled_agent_info.sled_agent_address,
+                REPO_DEPOT_PORT,
                 sled_agent_info.sled_role,
             );
             (*sled_id, sled)

dev-tools/reconfigurator-cli/tests/output/cmd-set-zone-images-stdout

@@ -312,6 +312,8 @@ internal DNS:
         AAAA fd00:1122:3344:101::26
     name: 805cba8f-a039-44e1-8cb0-1ff2c6b89997.host          (records: 1)
         AAAA fd00:1122:3344:101::23
+    name: 868d5b02-7792-4fc0-b6a9-654afcae9ea0.sled          (records: 1)
+        AAAA fd00:1122:3344:101::1
     name: _clickhouse-admin-single-server._tcp               (records: 1)
         SRV  port  8888 4fc6cb68-8ce9-4171-91b3-0106b8fed386.host.control-plane.oxide.internal
     name: _clickhouse-native._tcp                            (records: 1)
@@ -342,6 +344,8 @@ internal DNS:
         SRV  port 12221 296ab8b3-3fd4-4c2b-9e0a-1f909e4993e5.host.control-plane.oxide.internal
         SRV  port 12221 50dcee18-a857-4596-8cb7-fc9cf90b698d.host.control-plane.oxide.internal
         SRV  port 12221 805cba8f-a039-44e1-8cb0-1ff2c6b89997.host.control-plane.oxide.internal
+    name: _repo-depot._tcp                                   (records: 1)
+        SRV  port 12348 868d5b02-7792-4fc0-b6a9-654afcae9ea0.sled.control-plane.oxide.internal
     name: ac0f0ca6-02dd-43c7-b029-f34b247f7392.host          (records: 1)
         AAAA fd00:1122:3344:101::28
     name: ae74917b-2c6a-477d-9b90-fe2c2a4247f7.host          (records: 1)
@@ -580,6 +584,8 @@ internal DNS:
         AAAA fd00:1122:3344:101::26
     name: 805cba8f-a039-44e1-8cb0-1ff2c6b89997.host          (records: 1)
         AAAA fd00:1122:3344:101::23
+    name: 868d5b02-7792-4fc0-b6a9-654afcae9ea0.sled          (records: 1)
+        AAAA fd00:1122:3344:101::1
     name: _clickhouse-admin-single-server._tcp               (records: 1)
         SRV  port  8888 4fc6cb68-8ce9-4171-91b3-0106b8fed386.host.control-plane.oxide.internal
     name: _clickhouse-native._tcp                            (records: 1)
@@ -610,6 +616,8 @@ internal DNS:
         SRV  port 12221 296ab8b3-3fd4-4c2b-9e0a-1f909e4993e5.host.control-plane.oxide.internal
         SRV  port 12221 50dcee18-a857-4596-8cb7-fc9cf90b698d.host.control-plane.oxide.internal
         SRV  port 12221 805cba8f-a039-44e1-8cb0-1ff2c6b89997.host.control-plane.oxide.internal
+    name: _repo-depot._tcp                                   (records: 1)
+        SRV  port 12348 868d5b02-7792-4fc0-b6a9-654afcae9ea0.sled.control-plane.oxide.internal
     name: ac0f0ca6-02dd-43c7-b029-f34b247f7392.host          (records: 1)
         AAAA fd00:1122:3344:101::28
     name: ae74917b-2c6a-477d-9b90-fe2c2a4247f7.host          (records: 1)

internal-dns/types/src/names.rs

@@ -49,6 +49,7 @@ pub enum ServiceName {
     Nexus,
     Oximeter,
     ManagementGatewayService,
+    RepoDepot,
     Wicketd,
     Dendrite,
     Tfport,
@@ -80,6 +81,7 @@ impl ServiceName {
             ServiceName::Nexus => "nexus",
             ServiceName::Oximeter => "oximeter",
             ServiceName::ManagementGatewayService => "mgs",
+            ServiceName::RepoDepot => "repo-depot",
             ServiceName::Wicketd => "wicketd",
             ServiceName::Dendrite => "dendrite",
             ServiceName::Tfport => "tfport",
@@ -111,6 +113,7 @@ impl ServiceName {
             | ServiceName::Nexus
             | ServiceName::Oximeter
             | ServiceName::ManagementGatewayService
+            | ServiceName::RepoDepot
             | ServiceName::Wicketd
             | ServiceName::Dendrite
             | ServiceName::Tfport

nexus/db-model/src/sled.rs

@@ -149,7 +149,9 @@ impl From<Sled> for execution::Sled {
     fn from(sled: Sled) -> Self {
         Self::new(
             SledUuid::from_untyped_uuid(sled.id()),
+            sled.policy(),
             sled.address(),
+            *sled.repo_depot_port,
             if sled.is_scrimlet {
                 SledRole::Scrimlet
             } else {

nexus/reconfigurator/execution/src/dns.rs

@@ -343,6 +343,8 @@ mod test {
     use nexus_types::deployment::blueprint_zone_type;
     use nexus_types::external_api::params;
     use nexus_types::external_api::shared;
+    use nexus_types::external_api::views::SledPolicy;
+    use nexus_types::external_api::views::SledProvisionPolicy;
     use nexus_types::external_api::views::SledState;
     use nexus_types::identity::Resource;
     use nexus_types::internal_api::params::DnsConfigParams;
@@ -353,6 +355,7 @@ mod test {
     use omicron_common::address::IpRange;
     use omicron_common::address::Ipv6Subnet;
     use omicron_common::address::RACK_PREFIX;
+    use omicron_common::address::REPO_DEPOT_PORT;
     use omicron_common::address::SLED_PREFIX;
     use omicron_common::address::get_sled_address;
     use omicron_common::address::get_switch_zone_address;
@@ -733,7 +736,11 @@ mod test {
             .map(|(i, (sled_id, subnet))| {
                 let sled_info = Sled::new(
                     *sled_id,
+                    SledPolicy::InService {
+                        provision_policy: SledProvisionPolicy::Provisionable,
+                    },
                     get_sled_address(Ipv6Subnet::new(subnet.network())),
+                    REPO_DEPOT_PORT,
                     // The first two of these (arbitrarily) will be marked
                     // Scrimlets.
                     if i < 2 { SledRole::Scrimlet } else { SledRole::Gimlet },
@@ -759,7 +766,8 @@ mod test {
         //
         // 2. Every SRV record that we find should have a "target" that points
         //    to another name within the DNS configuration, and that name should
-        //    be one of the ones with a AAAA record pointing to an Omicron zone.
+        //    be one of the ones with a AAAA record pointing to either an
+        //    Omicron zone or a global zone.
         //
         // 3. There is at least one SRV record for each service that we expect
         //    to appear in the representative system that we're working with.
@@ -784,7 +792,7 @@ mod test {
             .collect();
         println!("omicron zones by IP: {:#?}", omicron_zones_by_ip);
 
-        // Check to see that the out-of-service zone was actually excluded
+        // Check to see that the out-of-service zone was actually excluded.
         assert!(
             omicron_zones_by_ip.values().all(|id| *id != out_of_service_id)
         );
@@ -805,6 +813,17 @@ mod test {
             })
             .collect();
 
+        // We also want a mapping from underlay IP to each sled global zone.
+        // In this case, the value is the sled id.
+        let mut all_sleds_by_ip: BTreeMap<_, _> = sleds_by_id
+            .keys()
+            .map(|sled_id| {
+                let sled_subnet = sleds_by_id.get(sled_id).unwrap().subnet();
+                let global_zone_ip = *get_sled_address(sled_subnet).ip();
+                (global_zone_ip, *sled_id)
+            })
+            .collect();
+
         // Prune the special boundary NTP DNS name out, collecting their IP
         // addresses, and build a list of expected SRV targets to ensure these
         // IPs show up both in the special boundary NTP DNS name and as their
@@ -872,9 +891,23 @@ mod test {
                     continue;
                 }
 
+                if let Some(sled_id) = all_sleds_by_ip.remove(addr) {
+                    println!(
+                        "IP {} found in DNS corresponds with global zone \
+                        for sled {}",
+                        addr, sled_id
+                    );
+                    expected_srv_targets.insert(format!(
+                        "{}.{}",
+                        name, blueprint_dns_zone.zone_name
+                    ));
+                    continue;
+                }
+
                 println!(
                     "note: found IP ({}) not corresponding to any \
-                    Omicron zone or switch zone (name {:?})",
+                    Omicron zone, switch zone, or global zone \
+                    (name {:?})",
                     addr, name
                 );
             }
@@ -924,6 +957,7 @@ mod test {
             ServiceName::CruciblePantry,
             ServiceName::BoundaryNtp,
             ServiceName::InternalNtp,
+            ServiceName::RepoDepot,
         ]);
 
         for (name, records) in &blueprint_dns_zone.records {

nexus/reconfigurator/execution/src/omicron_sled_config.rs

@@ -159,7 +159,10 @@ mod tests {
     use nexus_types::deployment::BlueprintZoneImageSource;
     use nexus_types::deployment::BlueprintZoneType;
     use nexus_types::deployment::blueprint_zone_type;
+    use nexus_types::external_api::views::SledPolicy;
+    use nexus_types::external_api::views::SledProvisionPolicy;
     use nexus_types::external_api::views::SledState;
+    use omicron_common::address::REPO_DEPOT_PORT;
     use omicron_common::api::external::Generation;
     use omicron_common::api::internal::shared::DatasetKind;
     use omicron_common::disk::CompressionAlgorithm;
@@ -194,7 +197,11 @@ mod tests {
             sim_sled_agent.id,
             Sled::new(
                 sim_sled_agent.id,
+                SledPolicy::InService {
+                    provision_policy: SledProvisionPolicy::Provisionable,
+                },
                 sim_sled_agent_addr,
+                REPO_DEPOT_PORT,
                 SledRole::Scrimlet,
             ),
         )]);

nexus/test-utils/src/lib.rs

@@ -308,6 +308,21 @@ impl RackInitRequestBuilder {
             .expect("Failed to set up DNS for {kind}");
     }
 
+    fn add_gz_service_to_dns(
+        &mut self,
+        sled_id: SledUuid,
+        address: SocketAddrV6,
+        service_name: ServiceName,
+    ) {
+        let sled = self
+            .internal_dns_config
+            .host_sled(sled_id, *address.ip())
+            .expect("Failed to set up DNS for GZ service");
+        self.internal_dns_config
+            .service_backend_sled(service_name, &sled, address.port())
+            .expect("Failed to set up DNS for GZ service");
+    }
+
     // Special handling of ClickHouse, which has multiple SRV records for its
     // single zone.
     fn add_clickhouse_to_dns(
@@ -1036,6 +1051,17 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         .await
         .expect("Failed to start sled agent");
 
+        // Add a DNS entry for the TUF Repo Depot on this simulated sled agent.
+        let SocketAddr::V6(server_addr_v6) = sled_agent.repo_depot_address
+        else {
+            panic!("expected sim sled agent to be listening on IPv6");
+        };
+        self.rack_init_builder.add_gz_service_to_dns(
+            sled_id,
+            server_addr_v6,
+            ServiceName::RepoDepot,
+        );
+
         self.sled_agents.push(ControlPlaneTestContextSledAgent {
             _storage: tempdir,
             server: sled_agent,

nexus/types/src/deployment/execution/dns.rs

@@ -13,7 +13,7 @@ use omicron_uuid_kinds::SledUuid;
 
 use crate::{
     deployment::{
-        Blueprint, BlueprintZoneDisposition, BlueprintZoneType,
+        Blueprint, BlueprintZoneDisposition, BlueprintZoneType, SledFilter,
         blueprint_zone_type,
     },
     internal_api::params::{DnsConfigZone, DnsRecord},
@@ -132,6 +132,27 @@ pub fn blueprint_internal_dns_config(
         )?;
     }
 
+    // For each sled to which we are supposed to be replicating artifacts,
+    // define DNS entries for the repo depot service.
+    //
+    // Consumers need to be careful in using these names since artifacts are not
+    // replicated synchronously or atomically to all instances.  That is: a
+    // consumer should be careful when fetching an artifact about whether they
+    // really can just pick any backend of this service or not.
+    for (sled_id, sled) in sleds_by_id {
+        if !sled.policy().matches(SledFilter::TufArtifactReplication) {
+            continue;
+        }
+
+        let dns_sled =
+            dns_builder.host_sled(*sled_id, *sled.sled_agent_address().ip())?;
+        dns_builder.service_backend_sled(
+            ServiceName::RepoDepot,
+            &dns_sled,
+            sled.repo_depot_address().port(),
+        )?;
+    }
+
     Ok(dns_builder.build_zone())
 }
 

nexus/types/src/deployment/execution/utils.rs

@@ -8,36 +8,57 @@ use nexus_sled_agent_shared::inventory::SledRole;
 use omicron_common::address::{Ipv6Subnet, SLED_PREFIX};
 use omicron_uuid_kinds::SledUuid;
 
-use crate::deployment::{
-    Blueprint, BlueprintZoneDisposition, BlueprintZoneType, blueprint_zone_type,
+use crate::{
+    deployment::{
+        Blueprint, BlueprintZoneDisposition, BlueprintZoneType,
+        blueprint_zone_type,
+    },
+    external_api::views::SledPolicy,
 };
 
 /// The minimal information needed to represent a sled in the context of
 /// blueprint execution.
 #[derive(Debug, Clone)]
 pub struct Sled {
     id: SledUuid,
+    policy: SledPolicy,
     sled_agent_address: SocketAddrV6,
+    repo_depot_port: u16,
     role: SledRole,
 }
 
 impl Sled {
     pub fn new(
         id: SledUuid,
+        policy: SledPolicy,
         sled_agent_address: SocketAddrV6,
+        repo_depot_port: u16,
         role: SledRole,
     ) -> Sled {
-        Sled { id, sled_agent_address, role }
+        Sled { id, policy, sled_agent_address, repo_depot_port, role }
     }
 
     pub fn id(&self) -> SledUuid {
         self.id
     }
 
+    pub fn policy(&self) -> SledPolicy {
+        self.policy
+    }
+
     pub fn sled_agent_address(&self) -> SocketAddrV6 {
         self.sled_agent_address
     }
 
+    pub fn repo_depot_address(&self) -> SocketAddrV6 {
+        SocketAddrV6::new(
+            *self.sled_agent_address().ip(),
+            self.repo_depot_port,
+            0,
+            0,
+        )
+    }
+
     pub fn subnet(&self) -> Ipv6Subnet<SLED_PREFIX> {
         Ipv6Subnet::<SLED_PREFIX>::new(*self.sled_agent_address.ip())
     }

sled-agent/src/rack_setup/plan/service.rs

@@ -22,7 +22,7 @@ use nexus_types::deployment::{
 };
 use omicron_common::address::{
     DENDRITE_PORT, DNS_HTTP_PORT, DNS_PORT, Ipv6Subnet, MGD_PORT, MGS_PORT,
-    NEXUS_INTERNAL_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS,
+    NEXUS_INTERNAL_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS, REPO_DEPOT_PORT,
     RSS_RESERVED_ADDRESSES, ReservedRackSubnet, SLED_PREFIX, get_sled_address,
     get_switch_zone_address,
 };
@@ -297,6 +297,20 @@ impl Plan {
         let mut dns_builder = DnsConfigBuilder::new();
         let mut svc_port_builder = ServicePortBuilder::new(config);
 
+        // All sleds get a DNS entry for Repo Depot.
+        for sled in sled_info.iter() {
+            let dns_sled = dns_builder
+                .host_sled(sled.sled_id, *sled.sled_address.ip())
+                .unwrap();
+            dns_builder
+                .service_backend_sled(
+                    ServiceName::RepoDepot,
+                    &dns_sled,
+                    REPO_DEPOT_PORT,
+                )
+                .unwrap();
+        }
+
         // Scrimlets get DNS records for running Dendrite.
         let scrimlets: Vec<_> =
             sled_info.iter().filter(|s| s.is_scrimlet).collect();