[monarch][supervision] Increase GetState::<ActorState> default timeout and make it configurable

Pull Request resolved: #1474

The current timeout of 1 second for `GetState::<ActorState>` for supervision was causing at least one test to fail. This diff makes the value configurable, and also increases the default to 30 seconds. The failing test now passes.

Example test failure that shows the supervision timeout: P1984316048
ghstack-source-id: 315174069

Differential Revision: [D84232284](https://our.internmc.facebook.com/intern/diff/D84232284/)

**NOTE FOR REVIEWERS**: This PR has internal Meta-specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D84232284/)!

Author: samlurye

hyperactor_mesh/src/v1/actor_mesh.rs

@@ -405,6 +405,7 @@ mod tests {
     use crate::v1::ActorMeshRef;
     use crate::v1::Name;
     use crate::v1::ProcMesh;
+    use crate::v1::proc_mesh::GET_ACTOR_STATE_MAX_IDLE;
     use crate::v1::testactor;
     use crate::v1::testing;
 
@@ -577,6 +578,9 @@ mod tests {
     async fn test_actor_states_with_process_exit() {
         hyperactor_telemetry::initialize_logging_for_test();
 
+        let config = hyperactor::config::global::lock();
+        let _guard = config.override_key(GET_ACTOR_STATE_MAX_IDLE, Duration::from_secs(1));
+
         let instance = testing::instance().await;
         // Listen for supervision events sent to the parent instance.
         let (supervision_port, mut supervision_receiver) =

hyperactor_mesh/src/v1/proc_mesh.rs

@@ -72,6 +72,9 @@ declare_attrs! {
     /// The maximum idle time between updates while spawning actor meshes.
     @meta(CONFIG_ENV_VAR = "HYPERACTOR_MESH_ACTOR_SPAWN_MAX_IDLE".to_string())
     pub attr ACTOR_SPAWN_MAX_IDLE: Duration = Duration::from_secs(30);
+
+    @meta(CONFIG_ENV_VAR = "HYPERACTOR_MESH_GET_ACTOR_STATE_MAX_IDLE".to_string())
+    pub attr GET_ACTOR_STATE_MAX_IDLE: Duration = Duration::from_secs(30);
 }
 
 /// A reference to a single [`hyperactor::Proc`].
@@ -545,7 +548,9 @@ impl ProcMeshRef {
             // the agent will be unresponsive.
             // We handle this by setting a timeout on the recv, and if we don't get a
             // message we assume the agent is dead and return a failed state.
-            let state = RealClock.timeout(Duration::from_secs(1), rx.recv()).await;
+            let state = RealClock
+                .timeout(config::global::get(GET_ACTOR_STATE_MAX_IDLE), rx.recv())
+                .await;
             if let Ok(state) = state {
                 // Handle non-timeout receiver error.
                 let state = state?;

python/tests/test_actor_error.py

@@ -747,7 +747,7 @@ async def test_supervision_with_proc_mesh_stopped(mesh, v1: bool) -> None:
 # TODO - re-enable after resolving T232206970
 @pytest.mark.oss_skip
 @pytest.mark.parametrize("v1", [True, False])
-@pytest.mark.timeout(30)
+@pytest.mark.timeout(60)
 async def test_supervision_with_sending_error(v1: bool) -> None:
     # Messages of length > this will cause a send error and a returned
     # undeliverable.