diff --git a/.envrc b/.envrc index 48df8e3c630..bb15c030860 100644 --- a/.envrc +++ b/.envrc @@ -5,6 +5,7 @@ PATH_add out/cockroachdb/bin PATH_add out/clickhouse PATH_add out/dendrite-stub/bin PATH_add out/mgd/root/opt/oxide/mgd/bin +PATH_add out/lldp/root/opt/oxide/bin if [ "$OMICRON_USE_FLAKE" = 1 ] && nix flake show &> /dev/null then diff --git a/Cargo.lock b/Cargo.lock index e17ba8d368c..29923e5e662 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2481,7 +2481,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=205b3ccf75b527ac7a565285fdcc0c78f4fcee95#205b3ccf75b527ac7a565285fdcc0c78f4fcee95" +source = "git+https://github.com/oxidecomputer/maghemite?rev=9e94d6b79560c2e4639cba432fb0ed600e9a3ff8#9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" dependencies = [ "oxnet", "progenitor 0.11.2", @@ -6124,7 +6124,7 @@ dependencies = [ [[package]] name = "lldpd-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/lldp#61479b6922f9112fbe1e722414d2b8055212cb12" +source = "git+https://github.com/oxidecomputer/lldp?rev=01b79698257c8d0414a2da6529e381c6c1a50893#01b79698257c8d0414a2da6529e381c6c1a50893" dependencies = [ "chrono", "futures", @@ -6142,7 +6142,7 @@ dependencies = [ [[package]] name = "lldpd-common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/lldp#61479b6922f9112fbe1e722414d2b8055212cb12" +source = "git+https://github.com/oxidecomputer/lldp?rev=01b79698257c8d0414a2da6529e381c6c1a50893#01b79698257c8d0414a2da6529e381c6c1a50893" dependencies = [ "anyhow", "dpd-client 0.1.0 (git+https://github.com/oxidecomputer/dendrite?branch=main)", @@ -6363,7 +6363,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=205b3ccf75b527ac7a565285fdcc0c78f4fcee95#205b3ccf75b527ac7a565285fdcc0c78f4fcee95" +source = "git+https://github.com/oxidecomputer/maghemite?rev=9e94d6b79560c2e4639cba432fb0ed600e9a3ff8#9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" dependencies = [ "chrono", "colored 3.0.0", @@ -11091,7 +11091,7 @@ dependencies = [ [[package]] name = "protocol" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/lldp#61479b6922f9112fbe1e722414d2b8055212cb12" +source = "git+https://github.com/oxidecomputer/lldp?rev=01b79698257c8d0414a2da6529e381c6c1a50893#01b79698257c8d0414a2da6529e381c6c1a50893" dependencies = [ "anyhow", "schemars 0.8.22", @@ -11442,7 +11442,7 @@ dependencies = [ [[package]] name = "rdb-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=205b3ccf75b527ac7a565285fdcc0c78f4fcee95#205b3ccf75b527ac7a565285fdcc0c78f4fcee95" +source = "git+https://github.com/oxidecomputer/maghemite?rev=9e94d6b79560c2e4639cba432fb0ed600e9a3ff8#9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" dependencies = [ "oxnet", "schemars 0.8.22", diff --git a/Cargo.toml b/Cargo.toml index 8b5be5bb8b9..25fa4403c06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -558,8 +558,8 @@ libfalcon = { git = "https://github.com/oxidecomputer/falcon", branch = "main" } libnvme = { git = "https://github.com/oxidecomputer/libnvme", rev = "dd5bb221d327a1bc9287961718c3c10d6bd37da0" } linear-map = "1.2.0" live-tests-macros = { path = "live-tests/macros" } -lldpd_client = { git = "https://github.com/oxidecomputer/lldp", package = "lldpd-client" } -lldp_protocol = { git = "https://github.com/oxidecomputer/lldp", package = "protocol" } +lldpd_client = { git = "https://github.com/oxidecomputer/lldp", package = "lldpd-client", rev = "01b79698257c8d0414a2da6529e381c6c1a50893" } +lldp_protocol = { git = "https://github.com/oxidecomputer/lldp", package = "protocol", rev = "01b79698257c8d0414a2da6529e381c6c1a50893" } macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" newtype_derive = "0.1.6" @@ -567,8 +567,8 @@ ntp-admin-api = { path = "ntp-admin/api" } ntp-admin-client = { path = "clients/ntp-admin-client" } ntp-admin-types = { path = "ntp-admin/types" } ntp-admin-types-versions = { path = "ntp-admin/types/versions" } -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "205b3ccf75b527ac7a565285fdcc0c78f4fcee95" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "205b3ccf75b527ac7a565285fdcc0c78f4fcee95" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" } multimap = "0.10.1" nexus-auth = { path = "nexus/auth" } nexus-background-task-interface = { path = "nexus/background-task-interface" } @@ -698,7 +698,7 @@ rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", rev = "f raw-cpuid = { git = "https://github.com/oxidecomputer/rust-cpuid.git", rev = "a4cf01df76f35430ff5d39dc2fe470bcb953503b" } rayon = "1.10" rcgen = "0.12.1" -rdb-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "205b3ccf75b527ac7a565285fdcc0c78f4fcee95" } +rdb-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" } reconfigurator-cli = { path = "dev-tools/reconfigurator-cli" } reedline = "0.40.0" ref-cast = "1.0" diff --git a/dev-tools/downloader/src/lib.rs b/dev-tools/downloader/src/lib.rs index 44fb340de28..c2a8df82b58 100644 --- a/dev-tools/downloader/src/lib.rs +++ b/dev-tools/downloader/src/lib.rs @@ -74,6 +74,9 @@ enum Target { /// Transceiver Control binary TransceiverControl, + + /// LLDP binary + LLDP, } #[derive(Parser)] @@ -136,6 +139,7 @@ pub async fn run_cmd(args: DownloadArgs) -> Result<()> { Target::Cockroach => downloader.download_cockroach().await, Target::Console => downloader.download_console().await, Target::DendriteStub => downloader.download_dendrite_stub().await, + Target::LLDP => downloader.download_lldp().await, Target::MaghemiteMgd => downloader.download_maghemite_mgd().await, Target::Softnpu => downloader.download_softnpu().await, Target::TransceiverControl => { @@ -870,6 +874,80 @@ impl Downloader<'_> { Ok(()) } + async fn download_lldp(&self) -> std::result::Result<(), anyhow::Error> { + let download_dir = self.output_dir.join("downloads"); + tokio::fs::create_dir_all(&download_dir).await?; + + let checksums_path = self.versions_dir.join("lldp_checksums"); + let [lldp_sha2, lldp_linux_sha2] = get_values_from_file( + ["CIDL_SHA256", "LINUX_SHA256"], + &checksums_path, + ) + .await?; + let commit_path = self.versions_dir.join("lldp_openapi_version"); + let [commit] = get_values_from_file(["COMMIT"], &commit_path).await?; + + let repo = "oxidecomputer/lldp"; + let base_url = format!("{BUILDOMAT_URL}/{repo}/image/{commit}"); + + let filename = "lldp.tar.gz"; + let tarball_path = download_dir.join(filename); + download_file_and_verify( + &self.log, + &tarball_path, + &format!("{base_url}/{filename}"), + ChecksumAlgorithm::Sha2, + &lldp_sha2, + ) + .await?; + unpack_tarball(&self.log, &tarball_path, &download_dir).await?; + + let destination_dir = self.output_dir.join("lldp"); + let _ = tokio::fs::remove_dir_all(&destination_dir).await; + tokio::fs::create_dir_all(&destination_dir).await?; + copy_dir_all( + &download_dir.join("root"), + &destination_dir.join("root"), + )?; + + let binary_dir = destination_dir.join("root/opt/oxide/bin"); + + match os_name()? { + Os::Linux => { + let filename = "lldpd"; + let path = download_dir.join(filename); + download_file_and_verify( + &self.log, + &path, + &format!( + "{BUILDOMAT_URL}/{repo}/linux/{commit}/{filename}" + ), + ChecksumAlgorithm::Sha2, + &lldp_linux_sha2, + ) + .await?; + set_permissions(&path, 0o755).await?; + tokio::fs::copy(path, binary_dir.join(filename)).await?; + } + Os::Mac => { + info!(self.log, "Building lldp from source for macOS"); + + let binaries = [("lldpd", &["--no-default-features"][..])]; + + let built_binaries = + self.build_from_git("lldp", &commit, &binaries).await?; + + // Copy built binary to binary_dir + let dest = binary_dir.join("lldp"); + tokio::fs::copy(&built_binaries[0], &dest).await?; + set_permissions(&dest, 0o755).await?; + } + Os::Illumos => (), + } + + Ok(()) + } + async fn download_maghemite_mgd(&self) -> Result<()> { let download_dir = self.output_dir.join("downloads"); tokio::fs::create_dir_all(&download_dir).await?; diff --git a/dev-tools/ls-apis/tests/api_dependencies.out b/dev-tools/ls-apis/tests/api_dependencies.out index 94197641c23..79ded48efe6 100644 --- a/dev-tools/ls-apis/tests/api_dependencies.out +++ b/dev-tools/ls-apis/tests/api_dependencies.out @@ -46,6 +46,8 @@ Downstairs Controller (debugging only) (client: dsc-client) Management Gateway Service (client: gateway-client) consumed by: dpd (dendrite/dpd) via 1 path + consumed by: lldpd (lldp/lldpd) via 1 path + consumed by: mgd (maghemite/mgd) via 1 path consumed by: omicron-nexus (omicron/nexus) via 4 paths consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path consumed by: wicketd (omicron/wicketd) via 3 paths diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 2738d6ff1fb..f915570374f 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -573,7 +573,7 @@ task: "bfd_manager" configured period: every s last completed activation: , triggered by started at (s ago) and ran for ms - last completion reported error: failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name("_dendrite._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN } + last completion reported error: failed to resolve addresses for Maghemite: proto error: no records found for Query { name: Name("_mgd._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN } task: "blueprint_planner" configured period: every m @@ -1141,7 +1141,7 @@ task: "bfd_manager" configured period: every s last completed activation: , triggered by started at (s ago) and ran for ms - last completion reported error: failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name("_dendrite._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN } + last completion reported error: failed to resolve addresses for Maghemite: proto error: no records found for Query { name: Name("_mgd._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN } task: "blueprint_planner" configured period: every m diff --git a/dev-tools/omicron-dev/src/main.rs b/dev-tools/omicron-dev/src/main.rs index 9fa5ac0fc05..a049b3ebdcf 100644 --- a/dev-tools/omicron-dev/src/main.rs +++ b/dev-tools/omicron-dev/src/main.rs @@ -36,6 +36,8 @@ impl OmicronDevApp { async fn exec(&self) -> Result<(), anyhow::Error> { match &self.command { OmicronDevCmd::RunAll(args) => args.exec().await, + OmicronDevCmd::RunMultiple(args) => args.exec().await, + OmicronDevCmd::Topology(args) => args.exec().await, } } } @@ -44,6 +46,10 @@ impl OmicronDevApp { enum OmicronDevCmd { /// Run a full simulated control plane RunAll(RunAllArgs), + /// Run multiple simulated control planes + RunMultiple(RunMultipleArgs), + /// Create an simulated topolgy from a file + Topology(TopologyArgs), } #[derive(Clone, Debug, Args)] @@ -59,24 +65,23 @@ struct RunAllArgs { nexus_config: Utf8PathBuf, } +trait Configurable { + fn nexus_config(&self) -> &Utf8PathBuf; +} + +impl Configurable for RunAllArgs { + fn nexus_config(&self) -> &Utf8PathBuf { + &self.nexus_config + } +} + impl RunAllArgs { async fn exec(&self) -> Result<(), anyhow::Error> { // Start a stream listening for SIGINT - let signals = - Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); - let mut signal_stream = signals.fuse(); + let mut signal_stream = start_stream(); // Read configuration. - let config_str = fs::read_to_string(&self.nexus_config)?; - let mut config: NexusConfig = toml::from_str(&config_str).context( - format!("parsing config: {}", self.nexus_config.as_str()), - )?; - config.pkg.log = dropshot::ConfigLogging::File { - // See LogContext::new(), - path: "UNUSED".to_string().into(), - level: dropshot::ConfigLoggingLevel::Trace, - if_exists: dropshot::ConfigLoggingIfExists::Fail, - }; + let mut config = read_config(self)?; if let Some(p) = self.nexus_listen_port { config @@ -90,7 +95,7 @@ impl RunAllArgs { println!("omicron-dev: setting up all services ... "); let cptestctx = nexus_test_utils::omicron_dev_setup_with_config::< omicron_nexus::Server, - >(&mut config, 0, self.gateway_config.clone()) + >(&mut config, 1, self.gateway_config.clone()) .await .context("error setting up services")?; @@ -178,6 +183,24 @@ impl RunAllArgs { location, ); } + for (location, dendrite) in cptestctx.dendrite.read().unwrap().iter() { + println!( + "omicron-dev: dendrite: http://[::1]:{} ({})", + dendrite.port, location, + ); + } + for (location, lldpd) in &cptestctx.lldpd { + println!( + "omicron-dev: lldp: http://[::1]:{} ({})", + lldpd.port, location, + ); + } + for (location, mgd) in &cptestctx.mgd { + println!( + "omicron-dev: maghemite: http://[::1]:{} ({})", + mgd.port, location, + ); + } println!( "omicron-dev: silo name: {}", cptestctx.silo_name, @@ -193,10 +216,246 @@ impl RunAllArgs { assert_eq!(caught_signal.unwrap(), SIGINT); eprintln!( "omicron-dev: caught signal, shutting down and removing \ - temporary directory" + temporary directory" ); cptestctx.teardown().await; + + Ok(()) + } +} + +fn start_stream() -> futures::stream::Fuse { + let signals = Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); + signals.fuse() +} + +fn read_config(args: &dyn Configurable) -> Result { + let config_str = fs::read_to_string(&args.nexus_config())?; + let mut config: NexusConfig = toml::from_str(&config_str) + .context(format!("parsing config: {}", args.nexus_config().as_str()))?; + config.pkg.log = dropshot::ConfigLogging::File { + // See LogContext::new(), + path: "UNUSED".to_string().into(), + level: dropshot::ConfigLoggingLevel::Trace, + if_exists: dropshot::ConfigLoggingIfExists::Fail, + }; + Ok(config) +} + +#[derive(Clone, Debug, Args)] +struct RunMultipleArgs { + /// Override the gateway server configuration file. + #[clap(long, default_value = DEFAULT_SP_SIM_CONFIG)] + gateway_config: Utf8PathBuf, + /// Override the nexus configuration file. + #[clap(long, default_value = DEFAULT_NEXUS_CONFIG)] + nexus_config: Utf8PathBuf, + /// Number of "racks" to launch + #[clap(long, default_value_t = 1)] + count: u8, +} + +impl Configurable for RunMultipleArgs { + fn nexus_config(&self) -> &Utf8PathBuf { + &self.nexus_config + } +} + +impl RunMultipleArgs { + async fn exec(&self) -> Result<(), anyhow::Error> { + // Start a stream listening for SIGINT + let mut signal_stream = start_stream(); + + // Read configuration. + let mut config = read_config(self)?; + + let mut contexts = vec![]; + + for n in 0..self.count { + config + .deployment + .dropshot_external + .dropshot + .bind_address + .set_ip("0.0.0.0".parse().unwrap()); + config + .deployment + .dropshot_external + .dropshot + .bind_address + .set_port(0); + + config.deployment.dropshot_internal.bind_address.set_port(0); + config.deployment.dropshot_lockstep.bind_address.set_port(0); + config.deployment.techport_external_server_port = 0; + + println!("\nomicron-dev: setting up all services for rack {n}... "); + let cptestctx = + nexus_test_utils::omicron_dev_setup_with_config::< + omicron_nexus::Server, + >(&mut config, 1, self.gateway_config.clone()) + .await + .context("error setting up services")?; + + println!("omicron-dev: Adding disks to first sled agent"); + + // This is how our integration tests are identifying that "disks exist" + // within the database. + // + // This inserts: + // - DEFAULT_ZPOOL_COUNT zpools, each of which contains: + // - A crucible dataset + // - A debug dataset + DiskTest::new(&cptestctx).await; + + println!("omicron-dev: services are running."); + + // Print out basic information about what was started. + // NOTE: The stdout strings here are not intended to be stable, but they + // are used by the test suite. + let addr = cptestctx.external_client.bind_address; + println!("omicron-dev: nexus external API: {:?}", addr); + println!( + "omicron-dev: nexus internal API: {:?}", + cptestctx.server.get_http_server_internal_address(), + ); + println!( + "omicron-dev: nexus lockstep API: {:?}", + cptestctx.server.get_http_server_lockstep_address(), + ); + println!( + "omicron-dev: cockroachdb pid: {}", + cptestctx.database.pid(), + ); + println!( + "omicron-dev: cockroachdb URL: {}", + cptestctx.database.pg_config() + ); + println!( + "omicron-dev: cockroachdb directory: {}", + cptestctx.database.temp_dir().display() + ); + println!( + "omicron-dev: clickhouse native addr: {}", + cptestctx.clickhouse.native_address(), + ); + println!( + "omicron-dev: clickhouse http addr: {}", + cptestctx.clickhouse.http_address(), + ); + println!( + "omicron-dev: internal DNS HTTP: http://{}", + cptestctx.internal_dns.dropshot_server.local_addr() + ); + println!( + "omicron-dev: internal DNS: {}", + cptestctx.internal_dns.dns_server.local_address() + ); + println!( + "omicron-dev: external DNS name: {}", + cptestctx.external_dns_zone_name, + ); + println!( + "omicron-dev: external DNS HTTP: http://{}", + cptestctx.external_dns.dropshot_server.local_addr() + ); + println!( + "omicron-dev: external DNS: {}", + cptestctx.external_dns.dns_server.local_address() + ); + println!( + "omicron-dev: e.g. `dig @{} -p {} {}.sys.{}`", + cptestctx.external_dns.dns_server.local_address().ip(), + cptestctx.external_dns.dns_server.local_address().port(), + cptestctx.silo_name, + cptestctx.external_dns_zone_name, + ); + for (location, gateway) in &cptestctx.gateway { + println!( + "omicron-dev: management gateway: {} ({})", + gateway.client.baseurl(), + location, + ); + } + for (location, dendrite) in + cptestctx.dendrite.read().unwrap().iter() + { + println!( + "omicron-dev: dendrite: http://[::1]:{} ({})", + dendrite.port, location, + ); + } + for (location, lldpd) in &cptestctx.lldpd { + println!( + "omicron-dev: lldp: http://[::1]:{} ({})", + lldpd.port, location, + ); + } + for (location, mgd) in &cptestctx.mgd { + println!( + "omicron-dev: maghemite: http://[::1]:{} ({})", + mgd.port, location, + ); + } + println!( + "omicron-dev: silo name: {}", + cptestctx.silo_name, + ); + println!( + "omicron-dev: privileged user name: {}", + cptestctx.user_name.as_ref(), + ); + println!( + "omicron-dev: privileged password: {}", + cptestctx.password + ); + contexts.push(cptestctx); + } + + // Wait for a signal. + let caught_signal = signal_stream.next().await; + assert_eq!(caught_signal.unwrap(), SIGINT); + eprintln!( + "omicron-dev: caught signal, shutting down and removing \ + temporary directory" + ); + + for context in contexts { + context.teardown().await; + } + Ok(()) } } + +#[derive(Clone, Debug, Args)] +struct TopologyArgs { + /// Override the gateway server configuration file. + #[clap(long, default_value = DEFAULT_SP_SIM_CONFIG)] + gateway_config: Utf8PathBuf, + /// Override the nexus configuration file. + #[clap(long, default_value = DEFAULT_NEXUS_CONFIG)] + nexus_config: Utf8PathBuf, + /// Number of "racks" to launch + #[clap(long)] + file: Utf8PathBuf, +} + +impl Configurable for TopologyArgs { + fn nexus_config(&self) -> &Utf8PathBuf { + &self.nexus_config + } +} + +impl TopologyArgs { + async fn exec(&self) -> Result<(), anyhow::Error> { + // Start a stream listening for SIGINT + // let mut signal_stream = start_stream(); + + // Read configuration. + // let mut config = read_config(self)?; + + unimplemented!() + } +} diff --git a/dev-tools/omicron-dev/tests/output/cmd-omicron-dev-noargs-stderr b/dev-tools/omicron-dev/tests/output/cmd-omicron-dev-noargs-stderr index 4d19049a326..7362d960b32 100644 --- a/dev-tools/omicron-dev/tests/output/cmd-omicron-dev-noargs-stderr +++ b/dev-tools/omicron-dev/tests/output/cmd-omicron-dev-noargs-stderr @@ -3,8 +3,10 @@ Tools for working with a local Omicron deployment Usage: omicron-dev Commands: - run-all Run a full simulated control plane - help Print this message or the help of the given subcommand(s) + run-all Run a full simulated control plane + run-multiple Run multiple simulated control planes + topology Create an simulated topolgy from a file + help Print this message or the help of the given subcommand(s) Options: -h, --help Print help diff --git a/env.sh b/env.sh index 6a84c35902a..1ced95c37bb 100644 --- a/env.sh +++ b/env.sh @@ -12,6 +12,7 @@ export PATH="$OMICRON_WS/out/cockroachdb/bin:$PATH" export PATH="$OMICRON_WS/out/clickhouse:$PATH" export PATH="$OMICRON_WS/out/dendrite-stub/bin:$PATH" export PATH="$OMICRON_WS/out/mgd/root/opt/oxide/mgd/bin:$PATH" +export PATH="$OMICRON_WS/out/lldp/root/opt/oxide/bin:$PATH" # if xtrace was set previously, do not unset it case $OLD_SHELL_OPTS in diff --git a/internal-dns/types/src/config.rs b/internal-dns/types/src/config.rs index d5bef144343..6f31b585d53 100644 --- a/internal-dns/types/src/config.rs +++ b/internal-dns/types/src/config.rs @@ -399,6 +399,7 @@ impl DnsConfigBuilder { dendrite_port: u16, mgs_port: u16, mgd_port: u16, + lldpd_port: u16, ) -> anyhow::Result<()> { let zone = self.host_dendrite(sled_id, switch_zone_ip)?; self.service_backend_zone(ServiceName::Dendrite, &zone, dendrite_port)?; @@ -407,7 +408,8 @@ impl DnsConfigBuilder { &zone, mgs_port, )?; - self.service_backend_zone(ServiceName::Mgd, &zone, mgd_port) + self.service_backend_zone(ServiceName::Mgd, &zone, mgd_port)?; + self.service_backend_zone(ServiceName::Lldpd, &zone, lldpd_port) } /// Higher-level shorthand for adding a Nexus zone with both its internal diff --git a/internal-dns/types/src/names.rs b/internal-dns/types/src/names.rs index 323b2aea07f..eb4c77a743c 100644 --- a/internal-dns/types/src/names.rs +++ b/internal-dns/types/src/names.rs @@ -76,8 +76,8 @@ pub enum ServiceName { Crucible(OmicronZoneUuid), BoundaryNtp, InternalNtp, - Maghemite, //TODO change to Dpd - maghemite has several services. Mgd, + Lldpd, } impl ServiceName { @@ -120,8 +120,8 @@ impl ServiceName { ServiceName::Crucible(_) => "crucible", ServiceName::BoundaryNtp => "boundary-ntp", ServiceName::InternalNtp => "internal-ntp", - ServiceName::Maghemite => "maghemite", ServiceName::Mgd => "mgd", + ServiceName::Lldpd => "lldpd", } } @@ -152,7 +152,7 @@ impl ServiceName { | ServiceName::CruciblePantry | ServiceName::BoundaryNtp | ServiceName::InternalNtp - | ServiceName::Maghemite + | ServiceName::Lldpd | ServiceName::Mgd => { format!("_{}._tcp", self.service_kind()) } diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 4885362ed36..3a0be6afdbe 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -268,7 +268,13 @@ pub struct DpdConfig { pub address: SocketAddr, } -/// Configuration for the `Dendrite` dataplane daemon. +/// Configuration for the `LLDP` daemon. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct LldpdConfig { + pub address: SocketAddr, +} + +/// Configuration for the `Maghemite` routing daemon. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct MgdConfig { pub address: SocketAddr, @@ -1003,6 +1009,9 @@ pub struct PackageConfig { /// `Dendrite` dataplane daemon configuration #[serde(default)] pub dendrite: HashMap, + /// `LLDP` daemon configuration + #[serde(default)] + pub lldpd: HashMap, /// Maghemite mgd daemon configuration #[serde(default)] pub mgd: HashMap, @@ -1215,6 +1224,8 @@ mod test { type = "from_dns" [dendrite.switch0] address = "[::1]:12224" + [lldpd.switch0] + address = "[::1]:12230" [mgd.switch0] address = "[::1]:4676" [initial_reconfigurator_config] @@ -1370,6 +1381,13 @@ mod test { .unwrap(), } )]), + lldpd: HashMap::from([( + SwitchLocation::Switch0, + LldpdConfig { + address: SocketAddr::from_str("[::1]:12230") + .unwrap(), + } + )]), mgd: HashMap::from([( SwitchLocation::Switch0, MgdConfig { diff --git a/nexus/reconfigurator/execution/src/test_utils.rs b/nexus/reconfigurator/execution/src/test_utils.rs index c606d384cb0..faf296cec6e 100644 --- a/nexus/reconfigurator/execution/src/test_utils.rs +++ b/nexus/reconfigurator/execution/src/test_utils.rs @@ -118,10 +118,12 @@ pub fn overridables_for_test( .unwrap() .port; let mgd_port = cptestctx.mgd.get(&switch_location).unwrap().port; + let lldpd_port = cptestctx.lldpd.get(&switch_location).unwrap().port; overrides.override_switch_zone_ip(sled_id, ip); overrides.override_dendrite_port(sled_id, dendrite_port); overrides.override_mgs_port(sled_id, mgs_port); overrides.override_mgd_port(sled_id, mgd_port); + overrides.override_lldpd_port(sled_id, lldpd_port); } overrides } diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index f47bc679cb5..258c3512750 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -1422,7 +1422,7 @@ mod tests { | ServiceName::Dendrite | ServiceName::Tfport | ServiceName::BoundaryNtp - | ServiceName::Maghemite + | ServiceName::Lldpd | ServiceName::Mgd => { out.insert(service, Err(QueryError::NoRecordsFound)); } diff --git a/nexus/src/app/background/tasks/bfd.rs b/nexus/src/app/background/tasks/bfd.rs index 8662124e25a..8d3b1c3c4e8 100644 --- a/nexus/src/app/background/tasks/bfd.rs +++ b/nexus/src/app/background/tasks/bfd.rs @@ -5,12 +5,8 @@ //! Background task for managing switch bidirectional forwarding detection //! (BFD) sessions. -use crate::app::{ - background::tasks::networking::build_mgd_clients, - switch_zone_address_mappings, -}; - use crate::app::background::BackgroundTask; +use crate::app::mgd_clients; use futures::FutureExt; use futures::future::BoxFuture; use internal_dns_resolver::Resolver; @@ -118,21 +114,23 @@ impl BackgroundTask for BfdManager { let mut current: HashSet = HashSet::new(); - let mappings = match switch_zone_address_mappings(&self.resolver, log).await { - Ok(mappings) => mappings, - Err(e) => { - error!(log, "failed to resolve addresses for Dendrite services"; "error" => %e); - return json!({ - "error": - format!( - "failed to resolve addresses for Dendrite services: {:#}", - e - ) - }); - }, - }; - - let mgd_clients = build_mgd_clients(mappings, log, &self.resolver).await; + let mgd_clients = match mgd_clients(&self.resolver, log).await + { + Ok(mappings) => mappings, + Err(e) => { + error!( + log, + "failed to resolve addresses for Maghemite"; + "error" => %e); + return json!({ + "error": + format!( + "failed to resolve addresses for Maghemite: {:#}", + e + ) + }); + }, + }; for (location, c) in &mgd_clients { let client_current = match c.get_bfd_peers().await { diff --git a/nexus/src/app/background/tasks/networking.rs b/nexus/src/app/background/tasks/networking.rs index 7ff50a5798d..767a870fb1c 100644 --- a/nexus/src/app/background/tasks/networking.rs +++ b/nexus/src/app/background/tasks/networking.rs @@ -6,47 +6,9 @@ use db::datastore::SwitchPortSettingsCombinedResult; use dpd_client::types::{ LinkCreate, LinkId, LinkSettings, PortFec, PortSettings, PortSpeed, TxEq, }; -use internal_dns_types::names::ServiceName; use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; use nexus_db_queries::db; -use omicron_common::{address::MGD_PORT, api::external::SwitchLocation}; -use std::{ - collections::HashMap, - net::{Ipv6Addr, SocketAddrV6}, -}; - -pub(crate) async fn build_mgd_clients( - mappings: HashMap, - log: &slog::Logger, - resolver: &internal_dns_resolver::Resolver, -) -> HashMap { - let mut clients: Vec<(SwitchLocation, mg_admin_client::Client)> = vec![]; - for (location, addr) in &mappings { - let port = match resolver.lookup_all_socket_v6(ServiceName::Mgd).await { - Ok(addrs) => { - let port_map: HashMap = addrs - .into_iter() - .map(|sockaddr| (*sockaddr.ip(), sockaddr.port())) - .collect(); - - *port_map.get(&addr).unwrap_or(&MGD_PORT) - } - Err(e) => { - error!(log, "failed to addresses"; "error" => %e); - MGD_PORT - } - }; - - let socketaddr = - std::net::SocketAddr::V6(SocketAddrV6::new(*addr, port, 0, 0)); - let client = mg_admin_client::Client::new( - format!("http://{}", socketaddr).as_str(), - log.clone(), - ); - clients.push((*location, client)); - } - clients.into_iter().collect::>() -} +use std::collections::HashMap; pub(crate) fn api_to_dpd_port_settings( settings: &SwitchPortSettingsCombinedResult, diff --git a/nexus/src/app/background/tasks/sync_switch_configuration.rs b/nexus/src/app/background/tasks/sync_switch_configuration.rs index 9a91dd28a7c..a10c33d87dc 100644 --- a/nexus/src/app/background/tasks/sync_switch_configuration.rs +++ b/nexus/src/app/background/tasks/sync_switch_configuration.rs @@ -6,10 +6,8 @@ //! to relevant management daemons (dendrite, mgd, sled-agent, etc.) use crate::app::{ - background::tasks::networking::{ - api_to_dpd_port_settings, build_mgd_clients, - }, - dpd_clients, switch_zone_address_mappings, + background::tasks::networking::api_to_dpd_port_settings, dpd_clients, + mgd_clients, switch_zone_address_mappings, }; use oxnet::Ipv4Net; use slog::{Logger, o}; @@ -393,8 +391,19 @@ impl BackgroundTask for SwitchPortSettingsManager { }; // TODO https://github.com/oxidecomputer/omicron/issues/5201 - // build mgd clients - let mgd_clients = build_mgd_clients(mappings, &log, &self.resolver).await; + let mgd_clients = match + mgd_clients(&self.resolver, &log).await + { + Ok(mappings) => mappings, + Err(e) => { + error!( + log, + "failed to resolve addresses for Maghemite"; + "error" => %e); + continue; + }, + }; + let port_list = match self.switch_ports(opctx, &log).await { Ok(value) => value, diff --git a/nexus/src/app/bfd.rs b/nexus/src/app/bfd.rs index 1ae958c20d4..026c85a85cb 100644 --- a/nexus/src/app/bfd.rs +++ b/nexus/src/app/bfd.rs @@ -14,7 +14,7 @@ impl super::Nexus { switch: SwitchLocation, ) -> Result { let mg_client: mg_admin_client::Client = self - .mg_clients() + .mgd_clients() .await .map_err(|e| { Error::internal_error(&format!("failed to get mg clients: {e}")) diff --git a/nexus/src/app/bgp.rs b/nexus/src/app/bgp.rs index 60f23d958d4..94e5f10dba8 100644 --- a/nexus/src/app/bgp.rs +++ b/nexus/src/app/bgp.rs @@ -104,9 +104,9 @@ impl super::Nexus { ) -> ListResultVec { opctx.authorize(authz::Action::Read, &authz::FLEET).await?; let mut result = Vec::new(); - for (switch, client) in &self.mg_clients().await.map_err(|e| { + for (switch, client) in &self.mgd_clients().await.map_err(|e| { external::Error::internal_error(&format!( - "failed to get mg clients: {e}" + "failed to get mgd clients: {e}" )) })? { let router_info = match client.read_routers().await { @@ -161,9 +161,9 @@ impl super::Nexus { ) -> LookupResult { opctx.authorize(authz::Action::Read, &authz::FLEET).await?; let mut result = BgpExported::default(); - for (switch, client) in &self.mg_clients().await.map_err(|e| { + for (switch, client) in &self.mgd_clients().await.map_err(|e| { external::Error::internal_error(&format!( - "failed to get mg clients: {e}" + "failed to get mgd clients: {e}" )) })? { let router_info = match client.read_routers().await { @@ -231,9 +231,9 @@ impl super::Nexus { opctx.authorize(authz::Action::Read, &authz::FLEET).await?; let mut result = Vec::new(); - for (switch, client) in &self.mg_clients().await.map_err(|e| { + for (switch, client) in &self.mgd_clients().await.map_err(|e| { external::Error::internal_error(&format!( - "failed to get mg clients: {e}" + "failed to get mgd clients: {e}" )) })? { let history = match client @@ -273,9 +273,9 @@ impl super::Nexus { ) -> ListResultVec { opctx.authorize(authz::Action::Read, &authz::FLEET).await?; let mut result = Vec::new(); - for (switch, client) in &self.mg_clients().await.map_err(|e| { + for (switch, client) in &self.mgd_clients().await.map_err(|e| { external::Error::internal_error(&format!( - "failed to get mg clients: {e}" + "failed to get mgd clients: {e}" )) })? { let mut imported: Vec = Vec::new(); diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index f9a8057958c..9c92486bfee 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -31,7 +31,6 @@ use nexus_mgs_updates::MgsUpdateDriver; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::ReconfiguratorConfigParam; use nexus_types::fm; -use omicron_common::address::MGD_PORT; use omicron_common::address::MGS_PORT; use omicron_common::api::external::ByteCount; use omicron_common::api::external::Error; @@ -1123,25 +1122,11 @@ impl Nexus { lldpd_clients(resolver, rack_id, &self.log).await } - pub(crate) async fn mg_clients( + pub(crate) async fn mgd_clients( &self, ) -> Result, String> { let resolver = self.resolver(); - let mappings = - switch_zone_address_mappings(resolver, &self.log).await?; - let mut clients: Vec<(SwitchLocation, mg_admin_client::Client)> = - vec![]; - for (location, addr) in &mappings { - let port = MGD_PORT; - let socketaddr = - std::net::SocketAddr::V6(SocketAddrV6::new(*addr, port, 0, 0)); - let client = mg_admin_client::Client::new( - format!("http://{}", socketaddr).as_str(), - self.log.clone(), - ); - clients.push((*location, client)); - } - Ok(clients.into_iter().collect::>()) + mgd_clients(resolver, &self.log).await } pub(crate) fn demo_sagas( @@ -1269,33 +1254,142 @@ pub(crate) async fn dpd_clients( Ok(mappings) } -// We currently ignore the rack_id argument here, as the shared -// switch_zone_address_mappings function doesn't allow filtering on the rack ID. -// Since we only have a single rack, this is OK for now. -// TODO: https://github.com/oxidecomputer/omicron/issues/1276 -// +/// Returns a mapping of clients for the Maghemite daemons of reachable switch zones. +/// If we are unable to communicate with the switch zone and determine the mapping +/// of SwitchLocation -> Zone Underlay Address, we omit an entry for that client. +pub(crate) async fn mgd_clients( + resolver: &internal_dns_resolver::Resolver, + log: &slog::Logger, +) -> Result, String> { + let mgd_socketaddrs = match resolver + .lookup_all_socket_v6(ServiceName::Mgd) + .await + { + Ok(addrs) => addrs, + Err(e) => { + error!(log, "failed to resolve addresses for Maghemite services"; "error" => %e); + return Err(e.to_string()); + } + }; + + let clients: Vec<(SocketAddrV6, mg_admin_client::Client)> = mgd_socketaddrs + .iter() + .map(|socket_addr| { + let client = mg_admin_client::Client::new( + &format!("http://{socket_addr}"), + log.new(o!( + "component" => "MgdClient" + )), + ); + + (*socket_addr, client) + }) + .collect(); + + let mut mappings: HashMap = + HashMap::new(); + + for (addr, client) in clients { + let switch_slot = match client.switch_identifiers().await { + Ok(response) => response.slot, + Err(e) => { + error!( + log, + "failed to determine switch slot for maghemite"; + "error" => %e, + "addr" => %addr, + ); + continue; + } + }; + + let location = match switch_slot { + Some(0) => SwitchLocation::Switch0, + Some(1) => SwitchLocation::Switch1, + Some(v) => { + warn!(log, "unexpected value for switch slot: {v}"); + continue; + } + None => { + warn!(log, "maghemite has not learned switch slot from MGS"); + continue; + } + }; + + mappings.insert(location, client); + } + + Ok(mappings) +} + /// Returns a mapping of clients for the LLDP daemons of reachable switch zones. /// If we are unable to communicate with the switch zone and determine the mapping /// of SwitchLocation -> Zone Underlay Address, we omit an entry for that client. pub(crate) async fn lldpd_clients( resolver: &internal_dns_resolver::Resolver, + // TODO: https://github.com/oxidecomputer/omicron/issues/1276 _rack_id: Uuid, log: &slog::Logger, ) -> Result, String> { - let mappings = switch_zone_address_mappings(resolver, log).await?; - let log = log.new(o!( "component" => "LldpdClient")); - let port = lldpd_client::default_port(); - let clients: HashMap = mappings + let lldpd_socketaddrs = match resolver + .lookup_all_socket_v6(ServiceName::Lldpd) + .await + { + Ok(addrs) => addrs, + Err(e) => { + error!(log, "failed to resolve addresses for LLDP services"; "error" => %e); + return Err(e.to_string()); + } + }; + + let clients: Vec<(SocketAddrV6, lldpd_client::Client)> = lldpd_socketaddrs .iter() - .map(|(location, addr)| { - let lldpd_client = lldpd_client::Client::new( - &format!("http://[{addr}]:{port}"), - log.clone(), + .map(|socket_addr| { + let client = lldpd_client::Client::new( + &format!("http://{socket_addr}"), + log.new(o!( + "component" => "LldpClient" + )), ); - (*location, lldpd_client) + + (*socket_addr, client) }) .collect(); - Ok(clients) + + let mut mappings: HashMap = + HashMap::new(); + + for (addr, client) in clients { + let switch_slot = match client.switch_identifiers().await { + Ok(response) => response.slot, + Err(e) => { + error!( + log, + "failed to determine switch slot for lldpd"; + "error" => %e, + "addr" => %addr, + ); + continue; + } + }; + + let location = match switch_slot { + Some(0) => SwitchLocation::Switch0, + Some(1) => SwitchLocation::Switch1, + Some(v) => { + warn!(log, "unexpected value for switch slot: {v}"); + continue; + } + None => { + warn!(log, "Lldpd has not learned switch slot from MGS"); + continue; + } + }; + + mappings.insert(location, client); + } + + Ok(mappings) } /// Look up Dendrite addresses in DNS then determine the switch location of diff --git a/nexus/test-utils/src/nexus_test.rs b/nexus/test-utils/src/nexus_test.rs index cd5c2f694a2..789d6e6bc79 100644 --- a/nexus/test-utils/src/nexus_test.rs +++ b/nexus/test-utils/src/nexus_test.rs @@ -115,6 +115,7 @@ pub struct ControlPlaneTestContext { pub gateway: BTreeMap, pub dendrite: RwLock>, + pub lldpd: HashMap, pub mgd: HashMap, pub external_dns_zone_name: String, pub external_dns: dns_server::TransientServer, @@ -249,6 +250,9 @@ impl ControlPlaneTestContext { for (_, mut mgd) in self.mgd { mgd.cleanup().await.unwrap(); } + for (_, mut lldpd) in self.lldpd { + lldpd.cleanup().await.unwrap(); + } self.logctx.cleanup_successful(); } } diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index d045fe79671..9a2fedc44bd 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -26,6 +26,7 @@ use internal_dns_types::names::ServiceName; use nexus_config::Database; use nexus_config::DpdConfig; use nexus_config::InternalDns; +use nexus_config::LldpdConfig; use nexus_config::MgdConfig; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; use nexus_config::NexusConfig; @@ -142,6 +143,7 @@ pub struct ControlPlaneStarter<'a, N: NexusServer> { pub dendrite: RwLock>, pub mgd: HashMap, + pub lldpd: HashMap, // NOTE: Only exists after starting Nexus, until external Nexus is // initialized. @@ -198,6 +200,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { producer: None, gateway: BTreeMap::new(), dendrite: RwLock::new(HashMap::new()), + lldpd: HashMap::new(), mgd: HashMap::new(), nexus_internal: None, nexus_internal_addr: None, @@ -441,12 +444,43 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { self.config.pkg.dendrite.insert(switch_location, config); } + pub async fn start_lldp(&mut self, switch_location: SwitchLocation) { + let log = &self.logctx.log; + debug!(log, "Starting LLDP for {switch_location}"); + let mgs = self.gateway.get(&switch_location).unwrap(); + let mgs_addr = + SocketAddrV6::new(Ipv6Addr::LOCALHOST, mgs.port, 0, 0).into(); + + let dpd_port = + self.dendrite.read().unwrap().get(&switch_location).unwrap().port; + + // Set up an instance of lldpd + let lldpd = + dev::lldp::LldpdInstance::start(0, dpd_port, Some(mgs_addr)) + .await + .unwrap(); + + let port = lldpd.port; + self.lldpd.insert(switch_location, lldpd); + let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, port, 0, 0); + + debug!(log, "lldp port is {port}"); + + let config = LldpdConfig { address: std::net::SocketAddr::V6(address) }; + self.config.pkg.lldpd.insert(switch_location, config); + } + pub async fn start_mgd(&mut self, switch_location: SwitchLocation) { let log = &self.logctx.log; debug!(log, "Starting mgd for {switch_location}"); + let mgs = self.gateway.get(&switch_location).unwrap(); + let mgs_addr = + SocketAddrV6::new(Ipv6Addr::LOCALHOST, mgs.port, 0, 0).into(); // Set up an instance of mgd - let mgd = dev::maghemite::MgdInstance::start(0).await.unwrap(); + let mgd = dev::maghemite::MgdInstance::start(0, Some(mgs_addr)) + .await + .unwrap(); let port = mgd.port; self.mgd.insert(switch_location, mgd); let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, port, 0, 0); @@ -483,6 +517,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { .port, self.gateway.get(&switch_location).unwrap().port, self.mgd.get(&switch_location).unwrap().port, + self.lldpd.get(&switch_location).unwrap().port, ) .unwrap() } @@ -1254,6 +1289,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { logctx: self.logctx, gateway: self.gateway, dendrite: RwLock::new(self.dendrite.into_inner().unwrap()), + lldpd: self.lldpd, mgd: self.mgd, external_dns_zone_name: self.external_dns_zone_name.unwrap(), external_dns: self.external_dns.unwrap(), @@ -1296,6 +1332,9 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { for (_, mut mgd) in self.mgd { mgd.cleanup().await.unwrap(); } + for (_, mut lldpd) in self.lldpd { + lldpd.cleanup().await.unwrap(); + } self.logctx.cleanup_successful(); } @@ -1627,6 +1666,12 @@ pub(crate) async fn setup_with_config_impl( builder.start_dendrite(SwitchLocation::Switch0).boxed() }), ), + ( + "start_lldpd_switch0", + Box::new(|builder| { + builder.start_lldp(SwitchLocation::Switch0).boxed() + }), + ), ( "start_mgd_switch0", Box::new(|builder| { @@ -1673,6 +1718,12 @@ pub(crate) async fn setup_with_config_impl( .boxed() }), ), + ( + "start_lldpd_switch1", + Box::new(|builder| { + builder.start_lldp(SwitchLocation::Switch1).boxed() + }), + ), ( "start_mgd_switch1", Box::new(|builder| { diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs index b748bbe2985..48bc3584e30 100644 --- a/nexus/tests/integration_tests/initialization.rs +++ b/nexus/tests/integration_tests/initialization.rs @@ -165,6 +165,11 @@ async fn test_nexus_boots_before_dendrite() { starter.start_dendrite(SwitchLocation::Switch1).await; info!(log, "Started Dendrite"); + info!(log, "Starting lldp"); + starter.start_lldp(SwitchLocation::Switch0).await; + starter.start_lldp(SwitchLocation::Switch1).await; + info!(log, "Started lldp"); + info!(log, "Starting mgd"); starter.start_mgd(SwitchLocation::Switch0).await; starter.start_mgd(SwitchLocation::Switch1).await; diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 0b16b3cfca3..a151d053dea 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -59,6 +59,8 @@ async fn test_setup<'a>( starter.start_gateway(SwitchLocation::Switch1, None, sp_conf).await; starter.start_dendrite(SwitchLocation::Switch0).await; starter.start_dendrite(SwitchLocation::Switch1).await; + starter.start_lldp(SwitchLocation::Switch0).await; + starter.start_lldp(SwitchLocation::Switch1).await; starter.start_mgd(SwitchLocation::Switch0).await; starter.start_mgd(SwitchLocation::Switch1).await; starter.populate_internal_dns().await; diff --git a/nexus/types/src/deployment/execution/dns.rs b/nexus/types/src/deployment/execution/dns.rs index a212db79bde..ed83720b2b7 100644 --- a/nexus/types/src/deployment/execution/dns.rs +++ b/nexus/types/src/deployment/execution/dns.rs @@ -158,6 +158,7 @@ pub fn blueprint_internal_dns_config( overrides.dendrite_port(scrimlet.id()), overrides.mgs_port(scrimlet.id()), overrides.mgd_port(scrimlet.id()), + overrides.lldpd_port(scrimlet.id()), )?; } diff --git a/nexus/types/src/deployment/execution/overridables.rs b/nexus/types/src/deployment/execution/overridables.rs index 881a7c49bdd..154902d1a8c 100644 --- a/nexus/types/src/deployment/execution/overridables.rs +++ b/nexus/types/src/deployment/execution/overridables.rs @@ -4,6 +4,7 @@ use omicron_common::address::DENDRITE_PORT; use omicron_common::address::Ipv6Subnet; +use omicron_common::address::LLDP_PORT; use omicron_common::address::MGD_PORT; use omicron_common::address::MGS_PORT; use omicron_common::address::SLED_PREFIX; @@ -31,6 +32,8 @@ pub struct Overridables { pub mgd_ports: BTreeMap, /// map: sled id -> IP address of the sled's switch zone pub switch_zone_ips: BTreeMap, + /// map: sled id -> TCP port on which that sled's LLDP is listening + pub lldpd_ports: BTreeMap, } pub static DEFAULT: LazyLock = @@ -67,6 +70,16 @@ impl Overridables { self.mgd_ports.get(&sled_id).copied().unwrap_or(MGD_PORT) } + /// Specify the TCP port on which this sled's LLDP is listening + pub fn override_lldpd_port(&mut self, sled_id: SledUuid, port: u16) { + self.lldpd_ports.insert(sled_id, port); + } + + /// Returns the TCP port on which this sled's LLDP is listening + pub fn lldpd_port(&self, sled_id: SledUuid) -> u16 { + self.lldpd_ports.get(&sled_id).copied().unwrap_or(LLDP_PORT) + } + /// Specify the IP address of this switch zone pub fn override_switch_zone_ip( &mut self, diff --git a/package-manifest.toml b/package-manifest.toml index 81556853f23..29de788d861 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -654,10 +654,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "205b3ccf75b527ac7a565285fdcc0c78f4fcee95" +source.commit = "9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "fa122dfaf77b1c060f19ca0e044f57044342c8c2b444595ca156ac8885852ebe" +source.sha256 = "4a8068799336e59dd1baf93e83c66c22b9fab7375950dbfddae55596eb9448bd" output.type = "tarball" [package.mg-ddm] @@ -670,10 +670,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "205b3ccf75b527ac7a565285fdcc0c78f4fcee95" +source.commit = "9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "14c8f3f77149c5aa217f56b57585eb1bcffc507d04910e84fc87c033cbe0ef39" +source.sha256 = "8f01f601b462314d309b01d3f13672b3f24966c4a2266213506543dcc1fc9644" output.type = "zone" output.intermediate_only = true @@ -685,10 +685,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "205b3ccf75b527ac7a565285fdcc0c78f4fcee95" +source.commit = "9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt -source.sha256 = "53576d28cae4304db61367d687580e9b96da23e647898b9034fdfa1603376b0c" +source.sha256 = "b9b7eb4fc04b539c432a5cbc978b53f77ad6942a17137ac2912971e178a68917" output.type = "zone" output.intermediate_only = true @@ -696,8 +696,8 @@ output.intermediate_only = true service_name = "lldp" source.type = "prebuilt" source.repo = "lldp" -source.commit = "61479b6922f9112fbe1e722414d2b8055212cb12" -source.sha256 = "8f988c0b0fa3ad4121ab0e825298601035e56c5c054bdc3a1dfb4d6c8fd5b300" +source.commit = "01b79698257c8d0414a2da6529e381c6c1a50893" +source.sha256 = "896e184232559a117aa91a7307ee77a9f441f5bea4296938c7fcdf24ef28788d" output.type = "zone" output.intermediate_only = true diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index bb758f357dd..caeb0911e57 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -26,6 +26,7 @@ use nexus_types::deployment::{ blueprint_zone_type, }; use nexus_types::external_api::views::SledState; +use omicron_common::address::LLDP_PORT; use omicron_common::address::{ DENDRITE_PORT, DNS_HTTP_PORT, DNS_PORT, Ipv6Subnet, MGD_PORT, MGS_PORT, NEXUS_INTERNAL_PORT, NEXUS_LOCKSTEP_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS, @@ -333,6 +334,7 @@ impl Plan { DENDRITE_PORT, MGS_PORT, MGD_PORT, + LLDP_PORT, ) .unwrap(); } diff --git a/test-utils/src/dev/lldp.rs b/test-utils/src/dev/lldp.rs new file mode 100644 index 00000000000..6144c302445 --- /dev/null +++ b/test-utils/src/dev/lldp.rs @@ -0,0 +1,216 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Tools for managing LLDP during development + +use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::Context; +use tempfile::TempDir; +use tokio::{ + fs::File, + io::{AsyncBufReadExt, BufReader}, + time::{Instant, sleep}, +}; + +/// Specifies the amount of time we will wait for `lldpd` to launch, +/// which is currently confirmed by watching `lldpd`'s log output +/// for a message specifying the address and port `lldpd` is listening on. +pub const LLDPD_TIMEOUT: Duration = Duration::new(5, 0); + +pub struct LldpdInstance { + /// Port number the mgd instance is listening on. This can be provided + /// manually, or dynamically determined if a value of 0 is provided. + pub port: u16, + /// Arguments provided to the `lldpd` cli command. + pub args: Vec, + /// Child process spawned by running `lldpd` + pub child: Option, + /// Temporary directory where logging output and other files generated by + /// `lldpd` are stored. + pub data_dir: Option, +} + +impl LldpdInstance { + pub async fn start( + mut port: u16, + dpd_port: u16, + mgs_address: Option, + ) -> Result { + let temp_dir = TempDir::new()?; + let listen_addr = + SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), port); + + let mut args = vec![ + "run".to_string(), + "--listen-addr".into(), + listen_addr.to_string(), + "--port".to_string(), + dpd_port.to_string(), + ]; + + if let Some(socket_addr) = mgs_address { + args.push("--mgs-addr".to_string()); + args.push(socket_addr.to_string()); + } + + let child = tokio::process::Command::new("lldpd") + .args(&args) + .stdin(Stdio::null()) + .stdout(Stdio::from(redirect_file( + temp_dir.path(), + "lldpd_stdout", + )?)) + .stderr(Stdio::from(redirect_file( + temp_dir.path(), + "lldpd_stderr", + )?)) + .spawn() + .with_context(|| { + format!("failed to spawn `lldpd` (with args: {:?})", &args) + })?; + + let child = Some(child); + + let temp_dir = temp_dir.keep(); + if port == 0 { + port = discover_port( + temp_dir.join("lldpd_stdout").display().to_string(), + ) + .await + .with_context(|| { + format!( + "failed to discover lldpd port from files in {}", + temp_dir.display() + ) + })?; + } + + Ok(Self { port, args, child, data_dir: Some(temp_dir) }) + } + + pub async fn cleanup(&mut self) -> Result<(), anyhow::Error> { + if let Some(mut child) = self.child.take() { + child.start_kill().context("Sending SIGKILL to child")?; + child.wait().await.context("waiting for child")?; + } + if let Some(dir) = self.data_dir.take() { + std::fs::remove_dir_all(&dir).with_context(|| { + format!("cleaning up temporary directory {}", dir.display()) + })?; + } + Ok(()) + } +} + +impl Drop for LldpdInstance { + fn drop(&mut self) { + if self.child.is_some() || self.data_dir.is_some() { + eprintln!( + "WARN: dropped LldpdInstance without cleaning it up first \ + (there may still be a child process running and a \ + temporary directory leaked)" + ); + if let Some(child) = self.child.as_mut() { + let _ = child.start_kill(); + } + if let Some(path) = self.data_dir.take() { + eprintln!( + "WARN: lldpd temporary directory leaked: {}", + path.display() + ); + } + } + } +} + +fn redirect_file( + temp_dir_path: &Path, + label: &str, +) -> Result { + let out_path = temp_dir_path.join(label); + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&out_path) + .with_context(|| format!("open \"{}\"", out_path.display())) +} + +async fn discover_port(logfile: String) -> Result { + let timeout = Instant::now() + LLDPD_TIMEOUT; + tokio::time::timeout_at(timeout, find_lldpd_port_in_log(logfile)) + .await + .context("time out while discovering lldpd port number")? +} + +async fn find_lldpd_port_in_log(logfile: String) -> Result { + let re = regex::Regex::new(r#""local_addr":"\[::1?\]:([0-9]+)""#).unwrap(); + let mut reader = BufReader::new(File::open(&logfile).await?); + let mut lines = reader.lines(); + loop { + match lines.next_line().await? { + Some(line) => { + if let Some(cap) = re.captures(&line) { + // unwrap on get(1) should be ok, since captures() returns + // `None` if there are no matches found + let port = cap.get(1).unwrap(); + let result = port.as_str().parse::()?; + return Ok(result); + } + } + None => { + sleep(Duration::from_millis(10)).await; + + // We might have gotten a partial line; close the file, reopen + // it, and start reading again from the beginning. + reader = BufReader::new(File::open(&logfile).await?); + lines = reader.lines(); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::find_lldpd_port_in_log; + use std::io::Write; + use std::process::Stdio; + use tempfile::NamedTempFile; + + const EXPECTED_PORT: u16 = 12230; + + #[tokio::test] + async fn test_lldpd_in_path() { + // With no arguments, we expect to see the default help message. + tokio::process::Command::new("lldpd") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("Cannot find 'lldpd' on PATH. Refer to README.md for installation instructions"); + } + + #[tokio::test] + async fn test_discover_local_listening_port() { + // Write some data to a fake log file + // This line is representative of the kind of output that lldpd currently logs + let line = r#"{"msg":"listening","v":0,"name":"lldpd","level":30,"time":"2025-12-23T00:07:09.226947807Z","hostname":"sled03","pid":10187,"local_addr":"[::]:12230","server_id": +"1","unit":"api-server"}"#; + let mut file = NamedTempFile::new().unwrap(); + writeln!(file, "A garbage line").unwrap(); + writeln!(file, "{}", line).unwrap(); + writeln!(file, "Another garbage line").unwrap(); + file.flush().unwrap(); + + assert_eq!( + find_lldpd_port_in_log(file.path().display().to_string()) + .await + .unwrap(), + EXPECTED_PORT + ); + } +} diff --git a/test-utils/src/dev/maghemite.rs b/test-utils/src/dev/maghemite.rs index 4c2d85df3ee..343b67c08aa 100644 --- a/test-utils/src/dev/maghemite.rs +++ b/test-utils/src/dev/maghemite.rs @@ -4,6 +4,7 @@ //! Tools for managing Maghemite during development +use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::Duration; @@ -35,10 +36,13 @@ pub struct MgdInstance { } impl MgdInstance { - pub async fn start(mut port: u16) -> Result { + pub async fn start( + mut port: u16, + mgs_address: Option, + ) -> Result { let temp_dir = TempDir::new()?; - let args = vec![ + let mut args = vec![ "run".to_string(), "--admin-addr".into(), "::1".into(), @@ -53,6 +57,11 @@ impl MgdInstance { uuid::Uuid::new_v4().to_string(), ]; + if let Some(socket_addr) = mgs_address { + args.push("--mgs-addr".to_string()); + args.push(socket_addr.to_string()); + } + let child = tokio::process::Command::new("mgd") .args(&args) .stdin(Stdio::null()) diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs index 5a37ac2549e..6e573d03c54 100644 --- a/test-utils/src/dev/mod.rs +++ b/test-utils/src/dev/mod.rs @@ -9,6 +9,7 @@ pub mod clickhouse; pub mod db; pub mod dendrite; pub mod falcon; +pub mod lldp; pub mod maghemite; pub mod poll; #[cfg(feature = "seed-gen")] diff --git a/tools/install_builder_prerequisites.sh b/tools/install_builder_prerequisites.sh index 646c764f8e9..40ad1fce2e2 100755 --- a/tools/install_builder_prerequisites.sh +++ b/tools/install_builder_prerequisites.sh @@ -223,6 +223,7 @@ retry xtask download \ console \ dendrite-stub \ maghemite-mgd \ + lldp \ transceiver-control # Validate the PATH: @@ -232,6 +233,8 @@ expected_in_path=( 'cockroach' 'clickhouse' 'dpd' + 'mgd' + 'lldpd' ) function show_hint diff --git a/tools/lldp_checksums b/tools/lldp_checksums new file mode 100644 index 00000000000..969df2e386f --- /dev/null +++ b/tools/lldp_checksums @@ -0,0 +1,2 @@ +CIDL_SHA256="896e184232559a117aa91a7307ee77a9f441f5bea4296938c7fcdf24ef28788d" +LINUX_SHA256="a19cb451baccfa6f442b04d4668ae5c991d51e198256516937751a66549bdabe" diff --git a/tools/lldp_openapi_version b/tools/lldp_openapi_version new file mode 100644 index 00000000000..79b5357a7b2 --- /dev/null +++ b/tools/lldp_openapi_version @@ -0,0 +1 @@ +COMMIT="01b79698257c8d0414a2da6529e381c6c1a50893" diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 65569a7a3ab..c4bcd2c981e 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1 +1 @@ -COMMIT="205b3ccf75b527ac7a565285fdcc0c78f4fcee95" +COMMIT="4a096d2fba90f46ce7c59318e0f4bb9b9d76ed20" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index 65569a7a3ab..bdd090ee327 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1 +1 @@ -COMMIT="205b3ccf75b527ac7a565285fdcc0c78f4fcee95" +COMMIT="9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 12228122372..47f1a2f3509 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="53576d28cae4304db61367d687580e9b96da23e647898b9034fdfa1603376b0c" -MGD_LINUX_SHA256="118a3bac35064cd231dfd123a3d6c7aa308fdf099b4ba045dcdfac6fbb4aae92" \ No newline at end of file +CIDL_SHA256="b9b7eb4fc04b539c432a5cbc978b53f77ad6942a17137ac2912971e178a68917" +MGD_LINUX_SHA256="0f2bbd3cb8dc061242f2a7735660fff483918c3fb928b0737c688420c78430ba"