[monarch][supervision] Increase GetState::<ActorState> default timeout and make it configurable

The current timeout of 1 second for `GetState::<ActorState>` for supervision was causing at least one test to fail. This diff makes the value configurable, and also increases the default to 30 seconds. The failing test now passes.

Example test failure that shows the supervision timeout: P1984316048

Differential Revision: [D84232284](https://our.internmc.facebook.com/intern/diff/D84232284/)

**NOTE FOR REVIEWERS**: This PR has internal Meta-specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D84232284/)!

ghstack-source-id: 315062680
Pull Request resolved: #1474

Author: samlurye

hyperactor_mesh/src/v1/proc_mesh.rs

@@ -72,6 +72,9 @@ declare_attrs! {
     /// The maximum idle time between updates while spawning actor meshes.
     @meta(CONFIG_ENV_VAR = "HYPERACTOR_MESH_ACTOR_SPAWN_MAX_IDLE".to_string())
     pub attr ACTOR_SPAWN_MAX_IDLE: Duration = Duration::from_secs(30);
+
+    @meta(CONFIG_ENV_VAR = "HYPERACTOR_MESH_GET_ACTOR_STATE_MAX_IDLE".to_string())
+    pub attr GET_ACTOR_STATE_MAX_IDLE: Duration = Duration::from_secs(30);
 }
 
 /// A reference to a single [`hyperactor::Proc`].
@@ -533,7 +536,9 @@ impl ProcMeshRef {
             // the agent will be unresponsive.
             // We handle this by setting a timeout on the recv, and if we don't get a
             // message we assume the agent is dead and return a failed state.
-            let state = RealClock.timeout(Duration::from_secs(1), rx.recv()).await;
+            let state = RealClock
+                .timeout(config::global::get(GET_ACTOR_STATE_MAX_IDLE), rx.recv())
+                .await;
             if let Ok(state) = state {
                 // Handle non-timeout receiver error.
                 let state = state?;