diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 7a562912df..96d8105c2c 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -456,7 +456,17 @@ impl RackInitRequestBuilder { } } -pub struct ControlPlaneTestContextBuilder<'a, N: NexusServer> { +/// Starts the control plane for tests and tools +/// +/// This helper manges the configuration and startup of all the control plane +/// components in a way where: +/// +/// - control planes started in this way are isolated from each other (in that +/// they don't know about each other, have separate data/databases, etc.) +/// - components are generally listening on localhost and pointed at each other +/// - most components are the real deal, though a few are simulated (notably +/// sled agent and SPs) +pub struct ControlPlaneStarter<'a, N: NexusServer> { pub config: &'a mut NexusConfig, test_name: &'a str, rack_init_builder: RackInitRequestBuilder, @@ -505,12 +515,10 @@ pub struct ControlPlaneTestContextBuilder<'a, N: NexusServer> { } type StepInitFn<'a, N> = Box< - dyn for<'b> FnOnce( - &'b mut ControlPlaneTestContextBuilder<'a, N>, - ) -> BoxFuture<'b, ()>, + dyn for<'b> FnOnce(&'b mut ControlPlaneStarter<'a, N>) -> BoxFuture<'b, ()>, >; -impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { +impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { pub fn new(test_name: &'a str, config: &'a mut NexusConfig) -> Self { let start_time = chrono::Utc::now(); let logctx = LogContext::new(test_name, &config.pkg.log); @@ -1773,10 +1781,9 @@ pub async fn omicron_dev_setup_with_config( extra_sled_agents: u16, gateway_config_file: Utf8PathBuf, ) -> Result> { - let builder = - ControlPlaneTestContextBuilder::::new("omicron-dev", config); + let starter = ControlPlaneStarter::::new("omicron-dev", config); - let log = &builder.logctx.log; + let log = &starter.logctx.log; debug!(log, "Ensuring seed tarball exists"); // Start up a ControlPlaneTestContext, which tautologically sets up @@ -1793,7 +1800,7 @@ pub async fn omicron_dev_setup_with_config( status.log(log, &seed_tar); Ok(setup_with_config_impl( - builder, + starter, PopulateCrdb::FromSeed { input_tar: seed_tar }, sim::SimMode::Auto, None, @@ -1813,9 +1820,9 @@ pub async fn test_setup_with_config( extra_sled_agents: u16, gateway_config_file: Utf8PathBuf, ) -> ControlPlaneTestContext { - let builder = ControlPlaneTestContextBuilder::::new(test_name, config); + let starter = ControlPlaneStarter::::new(test_name, config); setup_with_config_impl( - builder, + starter, PopulateCrdb::FromEnvironmentSeed, sim_mode, initial_cert, @@ -1827,7 +1834,7 @@ pub async fn test_setup_with_config( } async fn setup_with_config_impl( - mut builder: ControlPlaneTestContextBuilder<'_, N>, + mut starter: ControlPlaneStarter<'_, N>, populate: PopulateCrdb, sim_mode: sim::SimMode, initial_cert: Option, @@ -1838,7 +1845,7 @@ async fn setup_with_config_impl( const STEP_TIMEOUT: Duration = Duration::from_secs(600); // All setups will start with CRDB and clickhouse - builder + starter .init_with_steps( vec![ ( @@ -1860,7 +1867,7 @@ async fn setup_with_config_impl( // DNS and Nexus, but we currently don't use SMF to manage the services used in // the test context so we need to make the Nexus / DNS information available // to get the switch services working. - builder + starter .init_with_steps( vec![ ( @@ -1886,7 +1893,7 @@ async fn setup_with_config_impl( .await; if second_nexus { - builder + starter .init_with_steps( vec![( "configure_second_nexus", @@ -1904,7 +1911,7 @@ async fn setup_with_config_impl( // agent will be for switch1. let mgs_config = gateway_config_file.clone(); - builder + starter .init_with_steps( vec![ ( @@ -1948,7 +1955,7 @@ async fn setup_with_config_impl( .await; if extra_sled_agents > 0 { - builder + starter .init_with_steps( vec![ ( @@ -1997,7 +2004,7 @@ async fn setup_with_config_impl( // The first and second sled agents have special UUIDs, and any extra ones // after that are random. - builder + starter .init_with_steps( vec![( "start_sled1", @@ -2016,7 +2023,7 @@ async fn setup_with_config_impl( .await; if extra_sled_agents > 0 { - builder + starter .init_with_steps( vec![( "start_sled2", @@ -2036,7 +2043,7 @@ async fn setup_with_config_impl( } for index in 1..extra_sled_agents { - builder + starter .init_with_steps( vec![( "add_extra_sled_agent", @@ -2059,7 +2066,7 @@ async fn setup_with_config_impl( // agent. Afterwards, configure the sled agents and start the rest of the // the required services. - builder + starter .init_with_steps( vec![ ( @@ -2103,7 +2110,7 @@ async fn setup_with_config_impl( ) .await; - builder.build() + starter.build() } /// Starts a simulated sled agent diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs index 9f1a7bc60c..b748bbe298 100644 --- a/nexus/tests/integration_tests/initialization.rs +++ b/nexus/tests/integration_tests/initialization.rs @@ -6,7 +6,7 @@ use gateway_test_utils::setup::DEFAULT_SP_SIM_CONFIG; use nexus_config::Database; use nexus_config::InternalDns; use nexus_test_interface::NexusServer; -use nexus_test_utils::{ControlPlaneTestContextBuilder, load_test_config}; +use nexus_test_utils::{ControlPlaneStarter, load_test_config}; use omicron_common::address::MGS_PORT; use omicron_common::api::internal::shared::SwitchLocation; use std::collections::HashMap; @@ -18,22 +18,21 @@ use tokio::time::timeout; async fn test_nexus_boots_before_cockroach() { let mut config = load_test_config(); - let mut builder = - ControlPlaneTestContextBuilder::::new( - "test_nexus_boots_before_cockroach", - &mut config, - ); + let mut starter = ControlPlaneStarter::::new( + "test_nexus_boots_before_cockroach", + &mut config, + ); - let log = builder.logctx.log.new(o!("component" => "test")); + let log = starter.logctx.log.new(o!("component" => "test")); - builder + starter .start_gateway( SwitchLocation::Switch0, Some(MGS_PORT), DEFAULT_SP_SIM_CONFIG.into(), ) .await; - builder + starter .start_gateway( SwitchLocation::Switch1, None, @@ -41,28 +40,28 @@ async fn test_nexus_boots_before_cockroach() { ) .await; - builder.start_dendrite(SwitchLocation::Switch0).await; - builder.start_dendrite(SwitchLocation::Switch1).await; - builder.start_mgd(SwitchLocation::Switch0).await; - builder.start_mgd(SwitchLocation::Switch1).await; - builder.start_internal_dns().await; - builder.start_external_dns().await; + starter.start_dendrite(SwitchLocation::Switch0).await; + starter.start_dendrite(SwitchLocation::Switch1).await; + starter.start_mgd(SwitchLocation::Switch0).await; + starter.start_mgd(SwitchLocation::Switch1).await; + starter.start_internal_dns().await; + starter.start_external_dns().await; // Start Nexus, referencing the internal DNS system. // // This call won't return successfully until we can... // 1. Contact the internal DNS system to find Cockroach // 2. Contact Cockroach to ensure the database has been populated - builder.config.deployment.database = Database::FromDns; - builder.config.deployment.internal_dns = InternalDns::FromAddress { - address: builder + starter.config.deployment.database = Database::FromDns; + starter.config.deployment.internal_dns = InternalDns::FromAddress { + address: starter .internal_dns .as_ref() .expect("Must start Internal DNS before acquiring an address") .dns_server .local_address(), }; - let nexus_config = builder.config.clone(); + let nexus_config = starter.config.clone(); let nexus_log = log.clone(); let nexus_handle = tokio::task::spawn(async move { info!(nexus_log, "Test: Trying to start Nexus (internal)"); @@ -77,43 +76,42 @@ async fn test_nexus_boots_before_cockroach() { // This is necessary for the prior call to "start Nexus" to succeed. info!(log, "Starting CRDB"); let populate = true; - builder.start_crdb(populate).await; + starter.start_crdb(populate).await; info!(log, "Started CRDB"); info!(log, "Populating internal DNS records"); - builder.populate_internal_dns().await; + starter.populate_internal_dns().await; info!(log, "Populated internal DNS records"); // Now that Cockroach has started, we expect the request to succeed. nexus_handle.await.expect("Test: Task starting Nexus has failed"); - builder.teardown().await; + starter.teardown().await; } #[tokio::test] async fn test_nexus_boots_before_dendrite() { let mut config = load_test_config(); - let mut builder = - ControlPlaneTestContextBuilder::::new( - "test_nexus_boots_before_dendrite", - &mut config, - ); + let mut starter = ControlPlaneStarter::::new( + "test_nexus_boots_before_dendrite", + &mut config, + ); - let log = builder.logctx.log.new(o!("component" => "test")); + let log = starter.logctx.log.new(o!("component" => "test")); // Start MGS + Sim SP. This is needed for the Dendrite client initialization // inside of Nexus initialization. We must use MGS_PORT here because Nexus // hardcodes it. info!(&log, "Starting MGS"); - builder + starter .start_gateway( SwitchLocation::Switch0, None, DEFAULT_SP_SIM_CONFIG.into(), ) .await; - builder + starter .start_gateway( SwitchLocation::Switch1, None, @@ -123,33 +121,33 @@ async fn test_nexus_boots_before_dendrite() { info!(&log, "Started MGS"); let populate = true; - builder.start_crdb(populate).await; - builder.start_internal_dns().await; - builder.start_external_dns().await; + starter.start_crdb(populate).await; + starter.start_internal_dns().await; + starter.start_external_dns().await; // Start Nexus, referencing the internal DNS system. // // This call won't return successfully until we can... // 1. Contact the internal DNS system to find Dendrite // 2. Contact Dendrite - builder.config.deployment.database = Database::FromUrl { - url: builder + starter.config.deployment.database = Database::FromUrl { + url: starter .database .as_ref() .expect("Must start CRDB first") .pg_config() .clone(), }; - builder.config.pkg.dendrite = HashMap::new(); - builder.config.deployment.internal_dns = InternalDns::FromAddress { - address: builder + starter.config.pkg.dendrite = HashMap::new(); + starter.config.deployment.internal_dns = InternalDns::FromAddress { + address: starter .internal_dns .as_ref() .expect("Must start Internal DNS before acquiring an address") .dns_server .local_address(), }; - let nexus_config = builder.config.clone(); + let nexus_config = starter.config.clone(); let nexus_log = log.clone(); let nexus_handle = tokio::task::spawn(async move { info!(nexus_log, "Test: Trying to start Nexus (internal)"); @@ -163,77 +161,76 @@ async fn test_nexus_boots_before_dendrite() { // // This is necessary for the prior call to "start Nexus" to succeed. info!(log, "Starting Dendrite"); - builder.start_dendrite(SwitchLocation::Switch0).await; - builder.start_dendrite(SwitchLocation::Switch1).await; + starter.start_dendrite(SwitchLocation::Switch0).await; + starter.start_dendrite(SwitchLocation::Switch1).await; info!(log, "Started Dendrite"); info!(log, "Starting mgd"); - builder.start_mgd(SwitchLocation::Switch0).await; - builder.start_mgd(SwitchLocation::Switch1).await; + starter.start_mgd(SwitchLocation::Switch0).await; + starter.start_mgd(SwitchLocation::Switch1).await; info!(log, "Started mgd"); info!(log, "Populating internal DNS records"); - builder + starter .record_switch_dns( nexus_test_utils::SLED_AGENT_UUID.parse().unwrap(), SwitchLocation::Switch0, ) .await; - builder + starter .record_switch_dns( nexus_test_utils::SLED_AGENT2_UUID.parse().unwrap(), SwitchLocation::Switch1, ) .await; - builder.populate_internal_dns().await; + starter.populate_internal_dns().await; info!(log, "Populated internal DNS records"); // Now that Dendrite has started, we expect the request to succeed. nexus_handle.await.expect("Test: Task starting Nexus has failed"); - builder.teardown().await; + starter.teardown().await; } // Helper to ensure we perform the same setup for the positive and negative test // cases. async fn nexus_schema_test_setup( - builder: &mut ControlPlaneTestContextBuilder<'_, omicron_nexus::Server>, + starter: &mut ControlPlaneStarter<'_, omicron_nexus::Server>, ) { let populate = true; - builder.start_crdb(populate).await; - builder.start_internal_dns().await; - builder.start_external_dns().await; + starter.start_crdb(populate).await; + starter.start_internal_dns().await; + starter.start_external_dns().await; let sp_conf: camino::Utf8PathBuf = DEFAULT_SP_SIM_CONFIG.into(); - builder.start_gateway(SwitchLocation::Switch0, None, sp_conf.clone()).await; - builder.start_gateway(SwitchLocation::Switch1, None, sp_conf).await; - builder.start_dendrite(SwitchLocation::Switch0).await; - builder.start_dendrite(SwitchLocation::Switch1).await; - builder.start_mgd(SwitchLocation::Switch0).await; - builder.start_mgd(SwitchLocation::Switch1).await; - builder.populate_internal_dns().await; + starter.start_gateway(SwitchLocation::Switch0, None, sp_conf.clone()).await; + starter.start_gateway(SwitchLocation::Switch1, None, sp_conf).await; + starter.start_dendrite(SwitchLocation::Switch0).await; + starter.start_dendrite(SwitchLocation::Switch1).await; + starter.start_mgd(SwitchLocation::Switch0).await; + starter.start_mgd(SwitchLocation::Switch1).await; + starter.populate_internal_dns().await; } #[tokio::test] async fn test_nexus_boots_with_valid_schema() { let mut config = load_test_config(); - let mut builder = - ControlPlaneTestContextBuilder::::new( - "test_nexus_boots_with_valid_schema", - &mut config, - ); + let mut starter = ControlPlaneStarter::::new( + "test_nexus_boots_with_valid_schema", + &mut config, + ); - nexus_schema_test_setup(&mut builder).await; + nexus_schema_test_setup(&mut starter).await; assert!( - timeout(Duration::from_secs(60), builder.start_nexus_internal(),) + timeout(Duration::from_secs(60), starter.start_nexus_internal(),) .await .is_ok(), "Nexus should have started" ); - builder.teardown().await; + starter.teardown().await; } #[tokio::test] @@ -252,15 +249,14 @@ async fn test_nexus_does_not_boot_without_valid_schema() { config.pkg.tunables.load_timeout = Some(std::time::Duration::from_secs(5)); - let mut builder = - ControlPlaneTestContextBuilder::::new( - "test_nexus_does_not_boot_without_valid_schema", - &mut config, - ); + let mut starter = ControlPlaneStarter::::new( + "test_nexus_does_not_boot_without_valid_schema", + &mut config, + ); - nexus_schema_test_setup(&mut builder).await; + nexus_schema_test_setup(&mut starter).await; - builder.database + starter.database .as_ref() .expect("Should have started CRDB") .connect() @@ -274,7 +270,7 @@ async fn test_nexus_does_not_boot_without_valid_schema() { .await .expect("Failed to update schema"); - let err = builder + let err = starter .start_nexus_internal() .await .expect_err("Nexus should have failed to start"); @@ -284,7 +280,7 @@ async fn test_nexus_does_not_boot_without_valid_schema() { "Saw error: {err}" ); - builder.teardown().await; + starter.teardown().await; } } @@ -299,15 +295,14 @@ async fn test_nexus_does_not_boot_until_schema_updated() { let mut config = load_test_config(); - let mut builder = - ControlPlaneTestContextBuilder::::new( - "test_nexus_does_not_boot_until_schema_updated", - &mut config, - ); + let mut starter = ControlPlaneStarter::::new( + "test_nexus_does_not_boot_until_schema_updated", + &mut config, + ); - nexus_schema_test_setup(&mut builder).await; + nexus_schema_test_setup(&mut starter).await; - let crdb = builder + let crdb = starter .database .as_ref() .expect("Should have started CRDB") @@ -341,11 +336,11 @@ async fn test_nexus_does_not_boot_until_schema_updated() { }); assert!( - timeout(Duration::from_secs(60), builder.start_nexus_internal(),) + timeout(Duration::from_secs(60), starter.start_nexus_internal(),) .await .is_ok(), "Nexus should have started" ); - builder.teardown().await; + starter.teardown().await; } diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 77d501e0f8..8a50c863d1 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -20,7 +20,7 @@ use nexus_test_utils::sql::ColumnValue; use nexus_test_utils::sql::Row; use nexus_test_utils::sql::SqlEnum; use nexus_test_utils::sql::process_rows; -use nexus_test_utils::{ControlPlaneTestContextBuilder, load_test_config}; +use nexus_test_utils::{ControlPlaneStarter, load_test_config}; use omicron_common::api::internal::shared::SwitchLocation; use omicron_test_utils::dev::db::{Client, CockroachInstance}; use pretty_assertions::{assert_eq, assert_ne}; @@ -45,26 +45,24 @@ const SCHEMA_DIR: &'static str = async fn test_setup<'a>( config: &'a mut NexusConfig, name: &'static str, -) -> ControlPlaneTestContextBuilder<'a, omicron_nexus::Server> { - let mut builder = - ControlPlaneTestContextBuilder::::new( - name, config, - ); +) -> ControlPlaneStarter<'a, omicron_nexus::Server> { + let mut starter = + ControlPlaneStarter::::new(name, config); let populate = false; - builder.start_crdb(populate).await; + starter.start_crdb(populate).await; let schema_dir = Utf8PathBuf::from(SCHEMA_DIR); - builder.config.pkg.schema = Some(SchemaConfig { schema_dir }); - builder.start_internal_dns().await; - builder.start_external_dns().await; + starter.config.pkg.schema = Some(SchemaConfig { schema_dir }); + starter.start_internal_dns().await; + starter.start_external_dns().await; let sp_conf: Utf8PathBuf = DEFAULT_SP_SIM_CONFIG.into(); - builder.start_gateway(SwitchLocation::Switch0, None, sp_conf.clone()).await; - builder.start_gateway(SwitchLocation::Switch1, None, sp_conf).await; - builder.start_dendrite(SwitchLocation::Switch0).await; - builder.start_dendrite(SwitchLocation::Switch1).await; - builder.start_mgd(SwitchLocation::Switch0).await; - builder.start_mgd(SwitchLocation::Switch1).await; - builder.populate_internal_dns().await; - builder + starter.start_gateway(SwitchLocation::Switch0, None, sp_conf.clone()).await; + starter.start_gateway(SwitchLocation::Switch1, None, sp_conf).await; + starter.start_dendrite(SwitchLocation::Switch0).await; + starter.start_dendrite(SwitchLocation::Switch1).await; + starter.start_mgd(SwitchLocation::Switch0).await; + starter.start_mgd(SwitchLocation::Switch1).await; + starter.populate_internal_dns().await; + starter } // Attempts to apply an update as a transaction.