Log supervision events in Handler<SupervisionFailureMessage> (#1951)

Summary:
Pull Request resolved: #1951

As title.

Reviewed By: shayne-fletcher

Differential Revision: D87493413

fbshipit-source-id: 1cf22deb039367b886f81bbceabf69f3b7cebc68

Author: pzhan9

monarch_hyperactor/src/actor.rs

@@ -899,9 +899,13 @@ impl Handler<SupervisionFailureMessage> for PythonActor {
                         // TODO: We also don't want to deliver multiple supervision
                         // events from the same mesh if an earlier one is handled.
                         tracing::info!(
-                            "__supervise__ on {} handled a supervision event, not reporting any further: {}",
+                            name = "ActorMeshStatus",
+                            status = "SupervisionError::Handled",
+                            // only care about the event sender when the message is handled
+                            actor_name = message.actor_mesh_name,
+                            event = %message.event,
+                            "__supervise__ on {} handled a supervision event, not reporting any further",
                             cx.self_id(),
-                            message.event
                         );
                         Ok(())
                     } else {
@@ -913,6 +917,22 @@ impl Handler<SupervisionFailureMessage> for PythonActor {
 
                         // False -- we propagate the event onward, but update it with the fact that
                         // this actor is now the event creator.
+                        for (actor_name, status) in [
+                            (
+                                message.actor_mesh_name.as_str(),
+                                "SupervisionError::Unhandled",
+                            ),
+                            (cx.self_id().name(), "UnhandledSupervisionEvent"),
+                        ] {
+                            tracing::info!(
+                                name = "ActorMeshStatus",
+                                status,
+                                actor_name,
+                                event = %message.event,
+                                "__supervise__ on {} did not handle a supervision event, reporting to the next next owner",
+                                cx.self_id(),
+                            );
+                        }
                         let err = ActorErrorKind::UnhandledSupervisionEvent(Box::new(
                             ActorSupervisionEvent::new(
                                 cx.self_id().clone(),
@@ -932,6 +952,22 @@ impl Handler<SupervisionFailureMessage> for PythonActor {
                     // Include the event it was handling in the error message.
 
                     // Add to caused_by chain.
+                    for (actor_name, status) in [
+                        (
+                            message.actor_mesh_name.as_str(),
+                            "SupervisionError::__supervise__::exception",
+                        ),
+                        (cx.self_id().name(), "UnhandledSupervisionEvent"),
+                    ] {
+                        tracing::info!(
+                            name = "ActorMeshStatus",
+                            status,
+                            actor_name,
+                            event = %message.event,
+                            "__supervise__ on {} threw an exception",
+                            cx.self_id(),
+                        );
+                    }
                     let err = ActorErrorKind::UnhandledSupervisionEvent(Box::new(
                         ActorSupervisionEvent::new(
                             cx.self_id().clone(),

monarch_hyperactor/src/supervision.rs

@@ -24,7 +24,7 @@ create_exception!(
 
 #[derive(Clone, Debug, Serialize, Deserialize, Named, PartialEq, Bind, Unbind)]
 pub struct SupervisionFailureMessage {
-    pub mesh_name: String,
+    pub actor_mesh_name: String,
     pub rank: usize,
     pub event: ActorSupervisionEvent,
 }
@@ -54,7 +54,7 @@ impl MeshFailure {
 impl From<SupervisionFailureMessage> for MeshFailure {
     fn from(message: SupervisionFailureMessage) -> Self {
         Self {
-            mesh_name: message.mesh_name,
+            mesh_name: message.actor_mesh_name,
             rank: message.rank,
             event: message.event,
         }

monarch_hyperactor/src/v1/actor_mesh.rs

@@ -443,7 +443,7 @@ fn send_state_change<F>(
     // Send a notification to the owning actor of this mesh, if there is one.
     if let Some(owner) = owner {
         if let Err(error) = owner.send(SupervisionFailureMessage {
-            mesh_name: actor_mesh_name.to_string(),
+            actor_mesh_name: actor_mesh_name.to_string(),
             rank,
             event: event.clone(),
         }) {