[#5333 3/6] Region snapshot replacement start (#6294)

This commit adds a "region snapshot replacement start" background task
that will:

1) check for snapshots that are stored on disks that were expunged, and
   insert region snapshot replacement requests for them.

2) check if there are any region snapshot replacement requests in the
   `Requested` state and run the new "region snapshot replacement start"
   saga for them. This background task will also pick up manually
   requested region snapshot replacements.

Also in this commit is the "region snapshot replacement start saga",
which will transition a region snapshot replacement request from
`Requested` to `Allocating` and:

- allocate a new Region (with source set to Some so that a clone occurs)

- create a blank Volume that will be used to stash the snapshot to
delete later

- swap the snapshot being replaced with the new region that was cloned
from the others in the read-only region set

- update the region snapshot replacement request's state to
"ReplacementDone" and clearing the operating saga id

This represents the first step to be taken after a snapshot goes away:
allocate the replacement, and swap it in to the affected snapshot
volume. Once this is done, anything that uses the snapshot volume as a
read-only parent will no longer reference the expunged snapshot, and
will work without any issues.

Existing constructed Volumes running in a propolis or pantry context
will remain unmodified: a future commit will contain the saga that takes
care of performing the necessary read-only target replacements.

Author: jmpesp

Cargo.lock

@@ -307,6 +307,7 @@ crossterm = { version = "0.28.1", features = ["event-stream"] }
 crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "e58ca3693cb9ce0438947beba10e97ee38a0966b" }
 crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "e58ca3693cb9ce0438947beba10e97ee38a0966b" }
 crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "e58ca3693cb9ce0438947beba10e97ee38a0966b" }
+crucible-common = { git = "https://github.com/oxidecomputer/crucible", rev = "e58ca3693cb9ce0438947beba10e97ee38a0966b" }
 csv = "1.3.0"
 curve25519-dalek = "4"
 datatest-stable = "0.2.9"

Cargo.toml

@@ -33,6 +33,7 @@ use nexus_saga_recovery::LastPass;
 use nexus_types::deployment::Blueprint;
 use nexus_types::internal_api::background::LookupRegionPortStatus;
 use nexus_types::internal_api::background::RegionReplacementDriverStatus;
+use nexus_types::internal_api::background::RegionSnapshotReplacementStartStatus;
 use nexus_types::inventory::BaseboardId;
 use omicron_uuid_kinds::CollectionUuid;
 use omicron_uuid_kinds::DemoSagaUuid;
@@ -1394,6 +1395,38 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
                 }
             }
         };
+    } else if name == "region_snapshot_replacement" {
+        match serde_json::from_value::<RegionSnapshotReplacementStartStatus>(
+            details.clone(),
+        ) {
+            Err(error) => eprintln!(
+                "warning: failed to interpret task details: {:?}: {:?}",
+                error, details
+            ),
+
+            Ok(status) => {
+                println!(
+                    "    total requests created ok: {}",
+                    status.requests_created_ok.len(),
+                );
+                for line in &status.requests_created_ok {
+                    println!("    > {line}");
+                }
+
+                println!(
+                    "    total start saga invoked ok: {}",
+                    status.start_invoked_ok.len(),
+                );
+                for line in &status.start_invoked_ok {
+                    println!("    > {line}");
+                }
+
+                println!("    errors: {}", status.errors.len());
+                for line in &status.errors {
+                    println!("    > {line}");
+                }
+            }
+        }
     } else {
         println!(
             "warning: unknown background task: {:?} \

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -127,6 +127,10 @@ task: "region_replacement_driver"
     drive region replacements forward to completion
 
 
+task: "region_snapshot_replacement_start"
+    detect if region snapshots need replacement and begin the process
+
+
 task: "saga_recovery"
     recovers sagas assigned to this Nexus
 
@@ -276,6 +280,10 @@ task: "region_replacement_driver"
     drive region replacements forward to completion
 
 
+task: "region_snapshot_replacement_start"
+    detect if region snapshots need replacement and begin the process
+
+
 task: "saga_recovery"
     recovers sagas assigned to this Nexus
 
@@ -412,6 +420,10 @@ task: "region_replacement_driver"
     drive region replacements forward to completion
 
 
+task: "region_snapshot_replacement_start"
+    detect if region snapshots need replacement and begin the process
+
+
 task: "saga_recovery"
     recovers sagas assigned to this Nexus