Typed global configuration from python integrated with the Attrs system (#1433)

Summary:
Pull Request resolved: #1433

This diff introduces a typed global configuration system in python for monarch, integrated with the hyperactor `Attrs` system and the `hyperactor::global::config` module. To make an attribute configurable via python, first add the meta attribute `PYTHON_CONFIG_KEY`, like so:
```
rootfoo(
    CONFIG_ENV_VAR = "HYPERACTOR_MESH_DEFAULT_TRANSPORT".to_string(),
    PYTHON_CONFIG_KEY = "default_transport".to_string(),
)
pub attr DEFAULT_TRANSPORT: ChannelTransport = ChannelTransport::Unix;
```
This makes it so that the `DEFAULT_TRANSPORT` attr can be configured using `monarch.configure(default_transport=...)`. In order to ensure that attrs with type `ChannelTransport` can be configured by passing `PyChannelTransport` values to `monarch.configure`, in `monarch_hyperactor/src/config.rs`, add the macro invocation:
```
declare_py_config_type!(PyChannelTransport as ChannelTransport);
```
For attrs with rust types that can be passed directly to and from python (like `String`), simply add:
```
declare_py_config_type!(String);
```
These macro invocations only need to be added once per unique attr type we want to support -- *not* once per actual attr.

From python, the `DEFAULT_TRANSPORT` attr is then configurable in the global config by calling
```
monarch._rust_bindings.monarch_hyperactor.config.configure(
    default_transport=ChannelTransport.MetaTlsWithHostname,
    ...
)
```
You can then get the currently configured values with
```
monarch._rust_bindings.monarch_hyperactor.config.get_configuration()
```
which returns
```
{
    "default_transport": ChannelTransport.MetaTlsWithHostname,
    ...
}
```
ghstack-source-id: 314206074
exported-using-ghexport

Reviewed By: mariusae

Differential Revision: D83701581

fbshipit-source-id: e41f7ccadf28ac20151ca842b6bb0ddfdc23d725

Author: samlurye

hyperactor/src/attrs.rs

@@ -135,6 +135,9 @@ pub struct AttrKeyInfo {
     pub parse: fn(&str) -> Result<Box<dyn SerializableValue>, anyhow::Error>,
     /// Default value for the attribute, if any.
     pub default: Option<&'static dyn SerializableValue>,
+    /// A reference to the relevant key object with the associated
+    /// type parameter erased. Can be downcast to a concrete Key<T>.
+    pub erased: &'static dyn ErasedKey,
 }
 
 inventory::collect!(AttrKeyInfo);
@@ -199,6 +202,39 @@ impl<T: 'static> Clone for Key<T> {
 
 impl<T: 'static> Copy for Key<T> {}
 
+/// A trait for type-erased keys.
+pub trait ErasedKey: Any + Send + Sync + 'static {
+    /// The name of the key.
+    fn name(&self) -> &'static str;
+
+    /// The typehash of the key's associated type.
+    fn typehash(&self) -> u64;
+
+    /// The typename of the key's associated type.
+    fn typename(&self) -> &'static str;
+}
+
+impl dyn ErasedKey {
+    /// Downcast a type-erased key to a specific key type.
+    pub fn downcast_ref<T: Named + 'static>(&'static self) -> Option<&'static Key<T>> {
+        (self as &dyn Any).downcast_ref::<Key<T>>()
+    }
+}
+
+impl<T: AttrValue> ErasedKey for Key<T> {
+    fn name(&self) -> &'static str {
+        self.name
+    }
+
+    fn typehash(&self) -> u64 {
+        T::typehash()
+    }
+
+    fn typename(&self) -> &'static str {
+        T::typename()
+    }
+}
+
 // Enable attr[key] syntax.
 impl<T: AttrValue> Index<Key<T>> for Attrs {
     type Output = T;
@@ -778,6 +814,7 @@ macro_rules! declare_attrs {
                     Ok(Box::new(value) as Box<dyn $crate::attrs::SerializableValue>)
                 },
                 default: Some($crate::paste! { &[<$name _DEFAULT>] }),
+                erased: &$name,
             }
         }
     };
@@ -830,6 +867,7 @@ macro_rules! declare_attrs {
                     Ok(Box::new(value) as Box<dyn $crate::attrs::SerializableValue>)
                 },
                 default: None,
+                erased: &$name,
             }
         }
     };

hyperactor/src/config.rs

@@ -48,6 +48,10 @@ declare_attrs! {
     /// key.
     pub attr CONFIG_ENV_VAR: String;
 
+    /// This is a meta-attribute specifying the name of the kwarg to pass to monarch.configure()
+    /// to set the attribute value in the global config.
+    pub attr PYTHON_CONFIG_KEY: String;
+
     /// Maximum frame length for codec
     @meta(CONFIG_ENV_VAR = "HYPERACTOR_CODEC_MAX_FRAME_LENGTH".to_string())
     pub attr CODEC_MAX_FRAME_LENGTH: usize = 10 * 1024 * 1024 * 1024; // 10 GiB

hyperactor/src/config/global.rs

@@ -229,13 +229,24 @@ pub fn get<T: AttrValue + Copy>(key: Key<T>) -> T {
 /// Default. Panics if the key has no default and is not set in
 /// any layer.
 pub fn get_cloned<T: AttrValue>(key: Key<T>) -> T {
+    try_get_cloned(key)
+        .expect("key must have a default")
+        .clone()
+}
+
+/// Try to get a key by cloning the value.
+///
+/// Resolution order: TestOverride -> Runtime -> Env -> File ->
+/// Default. Returns None if the key has no default and is not set in
+/// any layer.
+pub fn try_get_cloned<T: AttrValue>(key: Key<T>) -> Option<T> {
     let layers = LAYERS.read().unwrap();
     for layer in &layers.ordered {
         if layer.attrs.contains_key(key) {
-            return layer.attrs.get(key).unwrap().clone();
+            return layer.attrs.get(key).cloned();
         }
     }
-    key.default().expect("key must have a default").clone()
+    key.default().cloned()
 }
 
 /// Insert or replace a configuration layer for the given source.

hyperactor_mesh/src/proc_mesh.rs

@@ -33,6 +33,7 @@ use hyperactor::channel::ChannelAddr;
 use hyperactor::channel::ChannelTransport;
 use hyperactor::config;
 use hyperactor::config::CONFIG_ENV_VAR;
+use hyperactor::config::PYTHON_CONFIG_KEY;
 use hyperactor::context;
 use hyperactor::declare_attrs;
 use hyperactor::mailbox;
@@ -85,10 +86,12 @@ use std::sync::RwLock;
 
 declare_attrs! {
     /// Default transport type to use across the application.
-    @meta(CONFIG_ENV_VAR = "HYPERACTOR_MESH_DEFAULT_TRANSPORT".to_string())
-    attr DEFAULT_TRANSPORT: ChannelTransport = ChannelTransport::Unix;
+    @meta(
+        CONFIG_ENV_VAR = "HYPERACTOR_MESH_DEFAULT_TRANSPORT".to_string(),
+        PYTHON_CONFIG_KEY = "default_transport".to_string(),
+    )
+    pub attr DEFAULT_TRANSPORT: ChannelTransport = ChannelTransport::Unix;
 }
-
 /// Get the default transport type to use across the application.
 pub fn default_transport() -> ChannelTransport {
     config::global::get_cloned(DEFAULT_TRANSPORT)

monarch_hyperactor/src/channel.rs

@@ -13,6 +13,7 @@ use hyperactor::channel::ChannelTransport;
 use hyperactor::channel::MetaTlsAddr;
 use hyperactor::channel::TlsMode;
 use pyo3::exceptions::PyRuntimeError;
+use pyo3::exceptions::PyValueError;
 use pyo3::prelude::*;
 
 /// Python binding for [`hyperactor::channel::ChannelTransport`]
@@ -31,6 +32,33 @@ pub enum PyChannelTransport {
     // Sim(/*transport:*/ ChannelTransport), TODO kiuk@ add support
 }
 
+#[pymethods]
+impl PyChannelTransport {
+    fn get(&self) -> Self {
+        self.clone()
+    }
+}
+
+impl TryFrom<ChannelTransport> for PyChannelTransport {
+    type Error = PyErr;
+
+    fn try_from(transport: ChannelTransport) -> PyResult<Self> {
+        match transport {
+            ChannelTransport::Tcp => Ok(PyChannelTransport::Tcp),
+            ChannelTransport::MetaTls(TlsMode::Hostname) => {
+                Ok(PyChannelTransport::MetaTlsWithHostname)
+            }
+            ChannelTransport::MetaTls(TlsMode::IpV6) => Ok(PyChannelTransport::MetaTlsWithIpV6),
+            ChannelTransport::Local => Ok(PyChannelTransport::Local),
+            ChannelTransport::Unix => Ok(PyChannelTransport::Unix),
+            _ => Err(PyValueError::new_err(format!(
+                "unsupported transport: {}",
+                transport
+            ))),
+        }
+    }
+}
+
 #[pyclass(
     name = "ChannelAddr",
     module = "monarch._rust_bindings.monarch_hyperactor.channel"