[inventory] Add health check information for each sled (#9434)

This is the first PR for
#9412

This PR only implements checking SMF service health for each sled. There
will be follow up PRs that deal with adding the data types to the DB.

_NB: This PR is less than 1000 lines, not ~10,000. Most of the changes
here are API JSON additions_

### Manual testing on a simulated omicron

System has two unhealthy enabled services

```console
$ pfexec svcs -Za -H -o state,fmri,zone
<...>
maintenance    svc:/site/fake-service2:default                    global
maintenance    svc:/site/fake-service:default                     global
<...>

$ curl -H "api-version: 11.0.0"  http://[::1]:32824/inventory | jq
<...>
  "health_monitor": {
    "smf_services_in_maintenance": {
      "ok": {
        "services": [
          {
            "fmri": "svc:/site/fake-service2:default",
            "zone": "global"
          },
          {
            "fmri": "svc:/site/fake-service:default",
            "zone": "global"
          }
        ],
        "time_of_status": "2025-12-10T08:35:10.604488858Z"
      }
    }
  }
}
```

Disabling one of the unhealthy services should only show a single entry

```console
$ svcadm disable svc:/site/fake-service:default
$ pfexec svcs -Za -H -o state,fmri,zone
<...>
maintenance    svc:/site/fake-service2:default                    global
disabled       svc:/site/fake-service:default                     global
<...>
$ curl -H "api-version: 11.0.0"  http://[::1]:32824/inventory | jq
<...>
  "health_monitor": {
    "smf_services_in_maintenance": {
      "ok": {
        "services": [
          {
            "fmri": "svc:/site/fake-service2:default",
            "zone": "global"
          }
        ],
        "time_of_status": "2025-12-10T08:35:10.604488858Z"
      }
    }
  }
}
```

Disabling the other service should not return any services

```console
$ svcadm disable svc:/site/fake-service2:default
$ pfexec svcs -Za -H -o state,fmri,zone
<...>
disabled       svc:/site/fake-service2:default                    global
disabled       svc:/site/fake-service:default                     global
<...>
$ curl -H "api-version: 11.0.0"  http://[::1]:32824/inventory | jq
<...>
  "health_monitor": {
    "smf_services_in_maintenance": {
      "ok": {
        "services": [],
        "time_of_status": "2025-12-10T08:35:10.604488858Z"
      }
    }
  }
}
```

Author: karencfv

Cargo.lock

@@ -133,6 +133,7 @@ members = [
     "sled-agent/api",
     "sled-agent/bootstrap-agent-api",
     "sled-agent/config-reconciler",
+    "sled-agent/health-monitor",
     "sled-agent/repo-depot-api",
     "sled-agent/types",
     "sled-agent/types/versions",
@@ -300,6 +301,7 @@ default-members = [
     "sled-agent/api",
     "sled-agent/bootstrap-agent-api",
     "sled-agent/config-reconciler",
+    "sled-agent/health-monitor",
     "sled-agent/repo-depot-api",
     "sled-agent/types",
     "sled-agent/types/versions",
@@ -729,6 +731,7 @@ sled = "=0.34.7"
 sled-agent-api = { path = "sled-agent/api" }
 sled-agent-client = { path = "clients/sled-agent-client" }
 sled-agent-config-reconciler = { path = "sled-agent/config-reconciler" }
+sled-agent-health-monitor = { path = "sled-agent/health-monitor" }
 sled-agent-types = { path = "sled-agent/types" }
 sled-agent-types-versions = { path = "sled-agent/types/versions" }
 sled-agent-zone-images = { path = "sled-agent/zone-images" }

Cargo.toml

@@ -15,6 +15,7 @@ bhyve_api.workspace = true
 byteorder.workspace = true
 camino.workspace = true
 camino-tempfile.workspace = true
+chrono.workspace = true
 cfg-if.workspace = true
 crucible-smf.workspace = true
 debug-ignore.workspace = true
@@ -34,6 +35,8 @@ oxnet.workspace = true
 schemars.workspace = true
 serde.workspace = true
 slog.workspace = true
+slog-async.workspace = true
+slog-term.workspace = true
 slog-error-chain.workspace = true
 smf.workspace = true
 thiserror.workspace = true

illumos-utils/Cargo.toml

@@ -5,7 +5,7 @@
 //! Wrappers around illumos-specific commands.
 
 use dropshot::HttpError;
-use slog_error_chain::InlineErrorChain;
+use slog_error_chain::{InlineErrorChain, SlogInlineError};
 #[allow(unused)]
 use std::sync::atomic::{AtomicBool, Ordering};
 
@@ -28,6 +28,7 @@ pub mod scf;
 pub mod smf_helper;
 pub mod svc;
 pub mod svcadm;
+pub mod svcs;
 pub mod vmm_reservoir;
 pub mod zfs;
 pub mod zone;
@@ -58,7 +59,7 @@ impl std::fmt::Display for CommandFailureInfo {
     }
 }
 
-#[derive(thiserror::Error, Debug)]
+#[derive(thiserror::Error, Debug, SlogInlineError)]
 pub enum ExecutionError {
     #[error("Failed to start execution of [{command}]: {err}")]
     ExecutionStart { command: String, err: std::io::Error },