Support configurable heartbeat timeout (#698)

Summary:
Pull Request resolved: #698

Move RemoteAlloc HB timeout as part of configs

Monarch runs with 1k+ workers timeout on allocation with the default heartbeat setting (1s).

This is not a fix but simply extending configurability to Python side.

Reviewed By: vidhyav

Differential Revision: D79064585

fbshipit-source-id: f9a7857c087598951d80af94c393cb3d80e89103

Author: pablorfb-meta

hyperactor/src/config.rs

@@ -46,6 +46,9 @@ declare_attrs! {
 
     /// Timeout used by proc mesh for stopping an actor.
     pub attr STOP_ACTOR_TIMEOUT: Duration = Duration::from_secs(1);
+
+    /// Heartbeat interval for remote allocator
+    pub attr REMOTE_ALLOCATOR_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5);
 }
 
 /// Load configuration from environment variables
@@ -87,6 +90,13 @@ pub fn from_env() -> Attrs {
         }
     }
 
+    // Load remote allocator heartbeat interval
+    if let Ok(val) = env::var("HYPERACTOR_REMOTE_ALLOCATOR_HEARTBEAT_INTERVAL_SECS") {
+        if let Ok(parsed) = val.parse::<u64>() {
+            config[REMOTE_ALLOCATOR_HEARTBEAT_INTERVAL] = Duration::from_secs(parsed);
+        }
+    }
+
     config
 }
 
@@ -122,6 +132,9 @@ pub fn merge(config: &mut Attrs, other: &Attrs) {
     if other.contains_key(SPLIT_MAX_BUFFER_SIZE) {
         config[SPLIT_MAX_BUFFER_SIZE] = other[SPLIT_MAX_BUFFER_SIZE];
     }
+    if other.contains_key(REMOTE_ALLOCATOR_HEARTBEAT_INTERVAL) {
+        config[REMOTE_ALLOCATOR_HEARTBEAT_INTERVAL] = other[REMOTE_ALLOCATOR_HEARTBEAT_INTERVAL];
+    }
 }
 
 /// Global configuration functions
@@ -292,6 +305,10 @@ mod tests {
         );
         assert_eq!(config[MESSAGE_ACK_EVERY_N_MESSAGES], 1000);
         assert_eq!(config[SPLIT_MAX_BUFFER_SIZE], 5);
+        assert_eq!(
+            config[REMOTE_ALLOCATOR_HEARTBEAT_INTERVAL],
+            Duration::from_secs(5)
+        );
     }
 
     #[test]