include detailed proc status, command in ProcState when available (#1378)

Summary:
Pull Request resolved: #1378

Useful when debugging bootsrapping issues, and to show in error messages.
ghstack-source-id: 313885565
exported-using-ghexport

Reviewed By: shayne-fletcher

Differential Revision: D83535144

fbshipit-source-id: d6677bb8f46474eeb0a37ee55416b7ec37b595b7

Author: mariusae

hyperactor/src/host.rs

@@ -176,6 +176,11 @@ impl<M: ProcManager> Host<M> {
         Ok((host, frontend_handle))
     }
 
+    /// The underlying proc manager.
+    pub fn manager(&self) -> &M {
+        &self.manager
+    }
+
     /// The address which accepts messages destined for this host.
     pub fn addr(&self) -> &ChannelAddr {
         &self.frontend_addr

hyperactor_mesh/src/bootstrap.rs

@@ -418,7 +418,7 @@ pub fn install_pdeathsig_kill() -> io::Result<()> {
 /// In short:
 /// - `ProcState`/`ProcStopReason`: historical / event-driven model
 /// - `ProcStatus`: immediate status surface for lifecycle control
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub enum ProcStatus {
     /// The OS process has been spawned but is not yet fully running.
     /// (Process-level: child handle exists, no confirmation yet.)
@@ -427,7 +427,7 @@ pub enum ProcStatus {
     /// (Process-level: `pid` is known; Proc-level: bootstrap
     /// may still be running.)
     Running { pid: u32, started_at: SystemTime },
-    /// Ready means boostrap has completed and the proc is serving.
+    /// Ready means bootstrap has completed and the proc is serving.
     /// (Process-level: `pid` is known; Proc-level: bootstrap
     /// completed.)
     Ready {
@@ -1301,7 +1301,7 @@ impl hyperactor::host::ProcHandle for BootstrapProcHandle {
 }
 
 /// A specification of the command used to bootstrap procs.
-#[derive(Debug, Named, Serialize, Deserialize, Clone, Default)]
+#[derive(Debug, Named, Serialize, Deserialize, Clone, Default, PartialEq, Eq)]
 pub struct BootstrapCommand {
     pub program: std::path::PathBuf,
     pub arg0: Option<String>,
@@ -1430,10 +1430,15 @@ impl BootstrapProcManager {
         }
     }
 
+    /// The bootstrap command used to launch processes.
+    pub fn command(&self) -> &BootstrapCommand {
+        &self.command
+    }
+
     /// Return the current [`ProcStatus`] for the given [`ProcId`], if
     /// the proc is known to this manager.
     ///
-    /// This queries the live [`BootstrapProcHandle`] stored in the
+    /// This querprocies the live [`BootstrapProcHandle`] stored in the
     /// manager's internal map. It provides an immediate snapshot of
     /// lifecycle state (`Starting`, `Running`, `Stopping`, `Stopped`,
     /// etc.).

hyperactor_mesh/src/v1/host_mesh/mesh_agent.rs

@@ -13,6 +13,7 @@ use std::fmt;
 use std::pin::Pin;
 
 use async_trait::async_trait;
+use enum_as_inner::EnumAsInner;
 use hyperactor::Actor;
 use hyperactor::ActorHandle;
 use hyperactor::ActorRef;
@@ -31,6 +32,7 @@ use hyperactor::host::LocalProcManager;
 use serde::Deserialize;
 use serde::Serialize;
 
+use crate::bootstrap;
 use crate::bootstrap::BootstrapCommand;
 use crate::bootstrap::BootstrapProcManager;
 use crate::proc_mesh::mesh_agent::ProcMeshAgent;
@@ -51,6 +53,7 @@ type ProcManagerSpawnFn = Box<dyn Fn(Proc) -> ProcManagerSpawnFuture + Send + Sy
 ///
 /// This abstraction lets the same `HostAgent` work across both
 /// out-of-process and in-process execution modes.
+#[derive(EnumAsInner)]
 pub enum HostAgentMode {
     Process(Host<BootstrapProcManager>),
     Local(Host<LocalProcManager<ProcManagerSpawnFn>>),
@@ -173,6 +176,8 @@ impl Handler<ShutdownHost> for HostMeshAgent {
 pub struct ProcState {
     pub proc_id: ProcId,
     pub mesh_agent: ActorRef<ProcMeshAgent>,
+    pub bootstrap_command: Option<BootstrapCommand>,
+    pub proc_status: Option<bootstrap::ProcStatus>,
 }
 
 #[async_trait]
@@ -182,13 +187,24 @@ impl Handler<resource::GetState<ProcState>> for HostMeshAgent {
         cx: &Context<Self>,
         get_state: resource::GetState<ProcState>,
     ) -> anyhow::Result<()> {
+        let manager = self
+            .host
+            .as_mut()
+            .expect("host")
+            .as_process()
+            .map(Host::manager);
         let state = match self.created.get(&get_state.name) {
             Some(Ok((proc_id, mesh_agent))) => resource::State {
                 name: get_state.name.clone(),
                 status: resource::Status::Running,
                 state: Some(ProcState {
                     proc_id: proc_id.clone(),
                     mesh_agent: mesh_agent.clone(),
+                    bootstrap_command: manager.map(|m| m.command().clone()),
+                    proc_status: match manager {
+                        Some(manager) => Some(manager.status(proc_id).await.unwrap()),
+                        None => None,
+                    },
                 }),
             },
             Some(Err(e)) => resource::State {
@@ -288,10 +304,13 @@ impl Handler<GetHostMeshAgent> for HostMeshAgentProcMeshTrampoline {
 
 #[cfg(test)]
 mod tests {
+    use std::assert_matches::assert_matches;
+
     use hyperactor::Proc;
     use hyperactor::channel::ChannelTransport;
 
     use super::*;
+    use crate::bootstrap::ProcStatus;
     use crate::resource::CreateOrUpdateClient;
     use crate::resource::GetStateClient;
 
@@ -326,21 +345,24 @@ mod tests {
                 .await
                 .unwrap()
         );
-        assert_eq!(
+        assert_matches!(
             host_agent.get_state(&client, name.clone()).await.unwrap(),
             resource::State {
-                name: name.clone(),
+                name: resource_name,
                 status: resource::Status::Running,
                 state: Some(ProcState {
                     // The proc itself should be direct addressed, with its name directly.
-                    proc_id: ProcId::Direct(host_addr.clone(), name.to_string()),
+                    proc_id,
                     // The mesh agent should run in the same proc, under the name
                     // "agent".
-                    mesh_agent: ActorRef::attest(
-                        ProcId::Direct(host_addr.clone(), name.to_string()).actor_id("agent", 0)
-                    ),
+                    mesh_agent,
+                    bootstrap_command,
+                    proc_status: Some(ProcStatus::Ready { pid: _, started_at: _, addr: _, agent: proc_status_mesh_agent}),
                 }),
-            }
+            } if name == resource_name
+              && proc_id == ProcId::Direct(host_addr.clone(), name.to_string())
+              && mesh_agent == ActorRef::attest(ProcId::Direct(host_addr.clone(), name.to_string()).actor_id("agent", 0)) && bootstrap_command == Some(BootstrapCommand::test())
+              && mesh_agent == proc_status_mesh_agent
         );
     }
 }