test v6 routes reach mgd, hard coded ports -> dns

Author: rcgoodfellow

nexus/src/app/background/tasks/bfd.rs

@@ -132,7 +132,7 @@ impl BackgroundTask for BfdManager {
                 },
             };
 
-            let mgd_clients = build_mgd_clients(mappings, log);
+            let mgd_clients = build_mgd_clients(mappings, log, &self.resolver).await;
 
             for (location, c) in &mgd_clients {
                 let client_current = match c.get_bfd_peers().await {

nexus/src/app/background/tasks/networking.rs

@@ -6,18 +6,37 @@ use db::datastore::SwitchPortSettingsCombinedResult;
 use dpd_client::types::{
     LinkCreate, LinkId, LinkSettings, PortFec, PortSettings, PortSpeed, TxEq,
 };
+use internal_dns_types::names::ServiceName;
 use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed};
 use nexus_db_queries::db;
 use omicron_common::{address::MGD_PORT, api::external::SwitchLocation};
-use std::{collections::HashMap, net::SocketAddrV6};
+use std::{
+    collections::HashMap,
+    net::{Ipv6Addr, SocketAddrV6},
+};
 
-pub(crate) fn build_mgd_clients(
+pub(crate) async fn build_mgd_clients(
     mappings: HashMap<SwitchLocation, std::net::Ipv6Addr>,
     log: &slog::Logger,
+    resolver: &internal_dns_resolver::Resolver,
 ) -> HashMap<SwitchLocation, mg_admin_client::Client> {
     let mut clients: Vec<(SwitchLocation, mg_admin_client::Client)> = vec![];
     for (location, addr) in &mappings {
-        let port = MGD_PORT;
+        let port = match resolver.lookup_all_socket_v6(ServiceName::Mgd).await {
+            Ok(addrs) => {
+                let port_map: HashMap<Ipv6Addr, u16> = addrs
+                    .into_iter()
+                    .map(|sockaddr| (*sockaddr.ip(), sockaddr.port()))
+                    .collect();
+
+                *port_map.get(&addr).unwrap_or(&MGD_PORT)
+            }
+            Err(e) => {
+                error!(log, "failed to  addresses"; "error" => %e);
+                MGD_PORT
+            }
+        };
+
         let socketaddr =
             std::net::SocketAddr::V6(SocketAddrV6::new(*addr, port, 0, 0));
         let client = mg_admin_client::Client::new(

nexus/src/app/background/tasks/sync_switch_configuration.rs

@@ -335,6 +335,8 @@ impl BackgroundTask for SwitchPortSettingsManager {
         async move {
             let log = opctx.log.clone();
 
+            info!(log, "running switch port settings manager RPW");
+
             let racks = match self.datastore.rack_list_initialized(opctx, &DataPageParams::max_page()).await {
                 Ok(racks) => racks,
                 Err(e) => {
@@ -394,7 +396,7 @@ impl BackgroundTask for SwitchPortSettingsManager {
 
                 // TODO https://github.com/oxidecomputer/omicron/issues/5201
                 // build mgd clients
-                let mgd_clients = build_mgd_clients(mappings, &log);
+                let mgd_clients = build_mgd_clients(mappings, &log, &self.resolver).await;
 
                 let port_list = match self.switch_ports(opctx, &log).await {
                     Ok(value) => value,

nexus/src/app/mod.rs

@@ -1320,7 +1320,7 @@ async fn switch_zone_address_mappings(
             return Err(e.to_string());
         }
     };
-    Ok(map_switch_zone_addrs(&log, switch_zone_addresses).await)
+    Ok(map_switch_zone_addrs(&log, switch_zone_addresses, resolver).await)
 }
 
 // TODO: #3596 Allow updating of Nexus from `handoff_to_nexus()`
@@ -1344,14 +1344,33 @@ async fn switch_zone_address_mappings(
 async fn map_switch_zone_addrs(
     log: &Logger,
     switch_zone_addresses: Vec<Ipv6Addr>,
+    resolver: &internal_dns_resolver::Resolver,
 ) -> HashMap<SwitchLocation, Ipv6Addr> {
     use gateway_client::Client as MgsClient;
     info!(log, "Determining switch slots managed by switch zones");
     let mut switch_zone_addrs = HashMap::new();
 
     for addr in switch_zone_addresses {
+        let port = match resolver
+            .lookup_all_socket_v6(ServiceName::ManagementGatewayService)
+            .await
+        {
+            Ok(addrs) => {
+                let port_map: HashMap<Ipv6Addr, u16> = addrs
+                    .into_iter()
+                    .map(|sockaddr| (*sockaddr.ip(), sockaddr.port()))
+                    .collect();
+
+                *port_map.get(&addr).unwrap_or(&MGS_PORT)
+            }
+            Err(e) => {
+                error!(log, "failed to resolve MGS addresses"; "error" => %e);
+                MGS_PORT
+            }
+        };
+
         let mgs_client = MgsClient::new(
-            &format!("http://[{}]:{}", addr, MGS_PORT),
+            &format!("http://[{}]:{}", addr, port),
             log.new(o!("component" => "MgsClient")),
         );
 

nexus/src/app/switch_port.rs

@@ -94,7 +94,16 @@ impl super::Nexus {
         params: params::SwitchPortSettingsCreate,
         id: Option<Uuid>,
     ) -> CreateResult<SwitchPortSettingsCombinedResult> {
-        self.db_datastore.switch_port_settings_create(opctx, &params, id).await
+        let result = self
+            .db_datastore
+            .switch_port_settings_create(opctx, &params, id)
+            .await?;
+
+        // eagerly propagate changes via rpw
+        self.background_tasks
+            .activate(&self.background_tasks.task_switch_port_settings_manager);
+
+        Ok(result)
     }
 
     pub(crate) async fn switch_port_settings_update(

nexus/tests/integration_tests/switch_port.rs

@@ -20,7 +20,7 @@ use nexus_types::external_api::params::{
 use nexus_types::external_api::views::Rack;
 use omicron_common::api::external::{
     self, AddressLotKind, BgpPeer, IdentityMetadataCreateParams, LinkFec,
-    LinkSpeed, NameOrId, SwitchPort, SwitchPortSettings,
+    LinkSpeed, NameOrId, SwitchLocation, SwitchPort, SwitchPortSettings,
 };
 use omicron_common::api::external::{ImportExportPolicy, Name};
 use oxnet::IpNet;
@@ -400,7 +400,7 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) {
     .unwrap();
 }
 
-#[nexus_test]
+#[nexus_test(extra_sled_agents = 1)]
 async fn test_port_settings_basic_v6_crud(ctx: &ControlPlaneTestContext) {
     let client = &ctx.external_client;
 
@@ -429,9 +429,11 @@ async fn test_port_settings_basic_v6_crud(ctx: &ControlPlaneTestContext) {
     .unwrap();
 
     // Create port settings
+    let settings_name =
+        Name::from_str("nacelle").expect("nacell should be a valid name");
     let mut settings =
         SwitchPortSettingsCreate::new(IdentityMetadataCreateParams {
-            name: "nacelle".parse().unwrap(),
+            name: settings_name.clone(),
             description: "just a port".into(),
         });
 
@@ -499,4 +501,49 @@ async fn test_port_settings_basic_v6_crud(ctx: &ControlPlaneTestContext) {
     let route = &created.routes[0];
     assert_eq!(route.dst, IpNet::from_str("2000::/64").unwrap());
     assert_eq!(&route.gw.to_string(), "2000::1");
+
+    let mgd = &ctx.mgd[&SwitchLocation::Switch0];
+    let mgd_client = mg_admin_client::Client::new(
+        &format!("http://localhost:{}", mgd.port),
+        ctx.logctx.log.clone(),
+    );
+
+    // apply port settings
+    let apply_settings = SwitchPortApplySettings {
+        port_settings: NameOrId::Name(settings_name.clone()),
+    };
+
+    let racks_url = "/v1/system/hardware/racks";
+    let racks: Vec<Rack> =
+        NexusRequest::iter_collection_authn(client, racks_url, "", None)
+            .await
+            .expect("failed to list racks")
+            .all_items;
+
+    let rack_id = racks[0].identity.id;
+
+    NexusRequest::new(
+        RequestBuilder::new(
+            client,
+            Method::POST,
+            &format!("/v1/system/hardware/switch-port/qsfp0/settings?rack_id={rack_id}&switch_location=switch0"),
+        )
+        .body(Some(&apply_settings))
+        .expect_status(Some(StatusCode::NO_CONTENT)),
+    )
+    .authn_as(AuthnMode::PrivilegedUser)
+    .execute()
+    .await
+    .unwrap();
+
+    // wait for reconciliation
+    tokio::time::sleep(std::time::Duration::from_secs(10)).await;
+
+    // ensure that our route has landed in mgd
+    let routes = mgd_client
+        .static_list_v6_routes()
+        .await
+        .expect("list maghemite v6 routes");
+
+    assert_eq!(routes.len(), 1);
 }