[reconfigurator-cli] Extend target-release test to step through zone updates (#8940)

This is a step toward addressing #8478. It doesn't quite get there:

* I stopped at the point where we're ready to update Nexus; the current
behavior is wrong, and it didn't seem worth adding that wrong behavior
to this test.
* Our simulated system doesn't set up all zone types (it's missing
boundary NTP, cockroach, oximeter, and multinode clickhouse).

I'm not sure the second bullet is _super_ important for this test; we
don't really treat different zone types all that differently, other than
that some are expunge -> add and others are upgraded in place (and we do
have zones of both of those types in our simulated system). But I'd like
to leave #8478 open until we can address the first one.

In terms of review: the non-expectorate changes are quite small. I'd
recommend skimming the expectorate changes and checking the points where
there are comments about what just happened or is about to happen more
carefully. (As I was doing this I discovered that our simulated system
started with 3 pantry zones but during the upgrade test it expunged them
without replacing them; that's what led to changing the example system's
`target_crucible_pantry_zone_count`. So I think it is worth at least
skimming this output to see if there's anything else funky that I
missed.)

Author: jgallagher

dev-tools/reconfigurator-cli/src/lib.rs

@@ -245,7 +245,7 @@ fn process_command(
             cmd_sled_update_host_phase_2(sim, args)
         }
         Commands::SledUpdateRotBootloader(args) => {
-            cmd_sled_update_rot_bootlaoder(sim, args)
+            cmd_sled_update_rot_bootloader(sim, args)
         }
         Commands::SiloList => cmd_silo_list(sim),
         Commands::SiloAdd(args) => cmd_silo_add(sim, args),
@@ -402,6 +402,8 @@ struct SledSetArgs {
 enum SledSetCommand {
     /// set the policy for this sled
     Policy(SledSetPolicyArgs),
+    /// set the Omicron config for this sled from a blueprint
+    OmicronConfig(SledSetOmicronConfigArgs),
     #[clap(flatten)]
     Visibility(SledSetVisibilityCommand),
     /// set the mupdate override for this sled
@@ -415,6 +417,12 @@ struct SledSetPolicyArgs {
     policy: SledPolicyOpt,
 }
 
+#[derive(Debug, Args)]
+struct SledSetOmicronConfigArgs {
+    /// the blueprint to derive the Omicron config from
+    blueprint: BlueprintIdOpt,
+}
+
 #[derive(Debug, Subcommand)]
 enum SledSetVisibilityCommand {
     /// mark a sled hidden from inventory
@@ -1538,6 +1546,30 @@ fn cmd_sled_set(
             );
             Ok(Some(format!("set sled {sled_id} policy to {policy}")))
         }
+        SledSetCommand::OmicronConfig(command) => {
+            let resolved_id =
+                system.resolve_blueprint_id(command.blueprint.into())?;
+            let blueprint = system.get_blueprint(&resolved_id)?;
+            let sled_cfg =
+                blueprint.sleds.get(&sled_id).with_context(|| {
+                    format!("sled id {sled_id} not found in blueprint")
+                })?;
+            let omicron_sled_cfg =
+                sled_cfg.clone().into_in_service_sled_config();
+            system
+                .description_mut()
+                .sled_set_omicron_config(sled_id, omicron_sled_cfg)?;
+            sim.commit_and_bump(
+                format!(
+                    "reconfigurator-cli sled-set omicron-config: \
+                     {sled_id} from {resolved_id}",
+                ),
+                state,
+            );
+            Ok(Some(format!(
+                "set sled {sled_id} omicron config from {resolved_id}"
+            )))
+        }
         SledSetCommand::Visibility(command) => {
             let new = command.to_visibility();
             let prev = system
@@ -1636,7 +1668,7 @@ fn cmd_sled_update_install_dataset(
     )))
 }
 
-fn cmd_sled_update_rot_bootlaoder(
+fn cmd_sled_update_rot_bootloader(
     sim: &mut ReconfiguratorSim,
     args: SledUpdateRotBootloaderArgs,
 ) -> anyhow::Result<Option<String>> {

dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt

@@ -223,10 +223,195 @@ sled-update-host-phase2 d81c6a84-79b8-4958-ae41-ea46c9b19763 --boot-disk B --slo
 sled-update-host-phase1 d81c6a84-79b8-4958-ae41-ea46c9b19763 --active B --slot-b 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89
 inventory-generate
 
-# Do one more planning run. This should update one control plane zone.
+# Do another planning run. This should start updating zones (one at a time).
 blueprint-plan latest latest
 blueprint-diff latest
 
-# We should continue walking through the update. We need to build out a
-# reconfigurator-cli subcommand to simulate updated zone image sources (just
-# like we have sled-update-sp for simulated SP updates).
+# Update the first control plane zone and plan again, which should update the
+# next zone on this sled.
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+# Repeat until all non-Nexus zones on this sled have been updated. Some of these
+# steps update a zone in place; others expunge a zone and are followed by a zone
+# addition. It doesn't seem _particularly_ useful to spell out which step is
+# updating which zone, in terms of keeping this test up to date, but we will
+# have to tweak the number of times we iterate on this sled as our simulated
+# system or planner placement changes.
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+# The previous plan updated the last non-Nexus zone on this sled. Nexus comes
+# after everything else, so the next step should update the first zone on the
+# next sled.
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+# Step through updates of all the non-Nexus zones on this sled.
+sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+# The previous plan updated the last non-Nexus zone on this sled. Nexus comes
+# after everything else, so the next step should update the first zone on the
+# next sled.
+sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+# Step through updates of all the non-Nexus zones on this sled.
+sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+# The previous step updated the last non-Nexus zone on the final sled. We should
+# now see a blueprint where every in-service zone (other than Nexus) has an
+# image source set to an artifact from our TUF repo.
+blueprint-show latest
+
+# We ought to update the inventory for the final sled and then step through
+# the Nexus handoff process, but that work is still in progress.