Merge branch 'main' into lt/unify-bls

Author: ltitanb

.github/workflows/docs.yml

@@ -3,7 +3,7 @@ name: Docs
 on:
   push:
     branches:
-      - main
+      - stable
     # Review gh actions docs if you want to further define triggers, paths, etc
     # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#on
 

config.example.toml

@@ -2,7 +2,7 @@
 # Some fields are optional and can be omitted, in which case the default value, if present, will be used.
 
 # Chain spec ID. Supported values:
-# A network ID. Supported values: Mainnet, Holesky, Sepolia, Helder, Hoodi.
+# A network ID. Supported values: Mainnet, Holesky, Sepolia, Helder, Hoodi. Lower case values e.g. "mainnet" are also accepted
 # A custom object, e.g., chain = { genesis_time_secs = 1695902400, path = "/path/to/spec.json" }, with a path to a chain spec file, either in .json format (e.g., as returned by the beacon endpoint /eth/v1/config/spec), or in .yml format (see examples in tests/data).
 # A custom object, e.g., chain = { genesis_time_secs = 1695902400, slot_time_secs = 12, genesis_fork_version = "0x01017000" }.
 chain = "Holesky"
@@ -212,19 +212,21 @@ jwt_auth_fail_timeout_seconds = 300
 [signer.local.loader]
 # File: path to the keys file
 key_path = "./tests/data/keys.example.json"
-# ValidatorsDir: format of the keystore (lighthouse, prysm, teku or lodestar)
+# ValidatorsDir: format of the keystore (lighthouse, prysm, teku, lodestar, or nimbus)
 # format = "lighthouse"
 # ValidatorsDir: full path to the keys directory
-# For lighthouse, it's de path to the directory where the `<pubkey>/voting-keystore.json` directories are located.
+# For lighthouse, it's the path to the directory where the `<pubkey>` directories are located, under each of which is a `voting-keystore.json` file.
 # For prysm, it's the path to the `all-accounts.keystore.json` file.
 # For teku, it's the path to the directory where all `<pubkey>.json` files are located.
 # For lodestar, it's the path to the directory where all `<pubkey>.json` files are located.
+# For nimbus, it's the path to the directory where the `<pubkey>` directories are located, under each of which is a `keystore.json` file.
 # keys_path = ""
 # ValidatorsDir: full path to the secrets file/directory
-# For lighthouse, it's de path to the directory where the `<pubkey>.json` files are located.
+# For lighthouse, it's the path to the directory where the `<pubkey>` files are located.
 # For prysm, it's the path to the file containing the wallet decryption password.
 # For teku, it's the path to the directory where all `<pubkey>.txt` files are located.
 # For lodestar, it's the path to the file containing the decryption password.
+# For nimbus, it's the path to the directory where the `<pubkey>` files are located.
 # secrets_path = ""
 # Configuration for how the Signer module should store proxy delegations. Supported types of store are:
 #   - File: store keys and delegations from a plain text file (unsafe, use only for testing purposes)

crates/cli/src/docker_init.rs

@@ -256,7 +256,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
 
     if let Some(mux_config) = cb_config.muxes {
         for mux in mux_config.muxes.iter() {
-            if let Some((env_name, actual_path, internal_path)) = mux.loader_env() {
+            if let Some((env_name, actual_path, internal_path)) = mux.loader_env()? {
                 let (key, val) = get_env_val(&env_name, &internal_path);
                 pbs_envs.insert(key, val);
                 pbs_volumes.push(Volumes::Simple(format!("{}:{}:ro", actual_path, internal_path)));

crates/common/src/config/mod.rs

@@ -26,6 +26,7 @@ pub use utils::*;
 #[derive(Debug, Deserialize, Serialize)]
 pub struct CommitBoostConfig {
     pub chain: Chain,
+    #[serde(default)]
     pub relays: Vec<RelayConfig>,
     pub pbs: StaticPbsConfig,
     #[serde(flatten)]

crates/common/src/config/mux.rs

@@ -130,19 +130,29 @@ pub struct MuxConfig {
 
 impl MuxConfig {
     /// Returns the env, actual path, and internal path to use for the file
-    /// loader
-    pub fn loader_env(&self) -> Option<(String, String, String)> {
-        self.loader.as_ref().and_then(|loader| match loader {
+    /// loader. In File mode, validates the mux file prior to returning.   
+    pub fn loader_env(&self) -> eyre::Result<Option<(String, String, String)>> {
+        let Some(loader) = self.loader.as_ref() else {
+            return Ok(None);
+        };
+
+        match loader {
             MuxKeysLoader::File(path_buf) => {
-                let path =
-                    path_buf.to_str().unwrap_or_else(|| panic!("invalid path: {:?}", path_buf));
-                let internal_path = get_mux_path(&self.id);
+                let Some(path) = path_buf.to_str() else {
+                    bail!("invalid path: {:?}", path_buf);
+                };
+
+                let file = load_file(path)?;
+                // make sure we can load the pubkeys correctly
+                let _: Vec<BlsPublicKey> =
+                    serde_json::from_str(&file).wrap_err("failed to parse mux keys file")?;
 
-                Some((get_mux_env(&self.id), path.to_owned(), internal_path))
+                let internal_path = get_mux_path(&self.id);
+                Ok(Some((get_mux_env(&self.id), path.to_owned(), internal_path)))
             }
-            MuxKeysLoader::HTTP { .. } => None,
-            MuxKeysLoader::Registry { .. } => None,
-        })
+            MuxKeysLoader::HTTP { .. } => Ok(None),
+            MuxKeysLoader::Registry { .. } => Ok(None),
+        }
     }
 }
 

crates/common/src/config/pbs.rs

@@ -228,6 +228,14 @@ pub async fn load_pbs_config() -> Result<PbsModuleConfig> {
     let config = CommitBoostConfig::from_env_path()?;
     config.validate().await?;
 
+    // Make sure relays isn't empty - since the config is still technically valid if
+    // there are no relays for things like Docker compose generation, this check
+    // isn't in validate().
+    ensure!(
+        !config.relays.is_empty(),
+        "At least one relay must be configured to run the PBS service"
+    );
+
     // use endpoint from env if set, otherwise use default host and port
     let endpoint = if let Some(endpoint) = load_optional_env_var(PBS_ENDPOINT_ENV) {
         endpoint.parse()?

crates/common/src/signer/loader.rs

@@ -47,6 +47,8 @@ pub enum ValidatorKeysFormat {
     Lodestar,
     #[serde(alias = "prysm")]
     Prysm,
+    #[serde(alias = "nimbus")]
+    Nimbus,
 }
 
 impl SignerLoader {
@@ -85,6 +87,7 @@ impl SignerLoader {
                         load_from_lodestar_format(keys_path, secrets_path)
                     }
                     ValidatorKeysFormat::Prysm => load_from_prysm_format(keys_path, secrets_path),
+                    ValidatorKeysFormat::Nimbus => load_from_nimbus_format(keys_path, secrets_path),
                 };
             }
         })
@@ -275,6 +278,42 @@ fn load_from_prysm_format(
     Ok(signers)
 }
 
+fn load_from_nimbus_format(
+    keys_path: PathBuf,
+    secrets_path: PathBuf,
+) -> eyre::Result<Vec<ConsensusSigner>> {
+    let paths: Vec<_> =
+        fs::read_dir(&keys_path)?.map(|res| res.map(|e| e.path())).collect::<Result<_, _>>()?;
+
+    let signers = paths
+        .into_par_iter()
+        .filter_map(|path| {
+            if !path.is_dir() {
+                return None
+            }
+
+            let maybe_pubkey = path.file_name().and_then(|d| d.to_str())?;
+            let Ok(pubkey) = bls_pubkey_from_hex(maybe_pubkey) else {
+                warn!("Invalid pubkey: {}", maybe_pubkey);
+                return None
+            };
+
+            let ks_path = keys_path.join(maybe_pubkey).join("keystore.json");
+            let pw_path = secrets_path.join(pubkey.to_string());
+
+            match load_one(ks_path, pw_path) {
+                Ok(signer) => Some(signer),
+                Err(e) => {
+                    warn!("Failed to load signer for pubkey: {}, err: {}", pubkey, e);
+                    None
+                }
+            }
+        })
+        .collect();
+
+    Ok(signers)
+}
+
 fn load_one(ks_path: PathBuf, pw_path: PathBuf) -> eyre::Result<ConsensusSigner> {
     let keystore = Keystore::from_json_file(ks_path).map_err(|_| eyre!("failed reading json"))?;
     let password = fs::read(pw_path.clone())
@@ -305,7 +344,7 @@ mod tests {
     use super::{load_from_lighthouse_format, load_from_lodestar_format, FileKey};
     use crate::{
         signer::{
-            loader::{load_from_prysm_format, load_from_teku_format},
+            loader::{load_from_nimbus_format, load_from_prysm_format, load_from_teku_format},
             BlsSigner,
         },
         utils::bls_pubkey_from_hex_unchecked,
@@ -401,4 +440,16 @@ mod tests {
             "b3a22e4a673ac7a153ab5b3c17a4dbef55f7e47210b20c0cbb0e66df5b36bb49ef808577610b034172e955d2312a61b9"
         ));
     }
+
+    #[test]
+    fn test_load_nimbus() {
+        let result = load_from_nimbus_format(
+            "../../tests/data/keystores/nimbus-keys".into(),
+            "../../tests/data/keystores/secrets".into(),
+        );
+
+        assert!(result.is_ok());
+
+        test_correct_load(result.unwrap());
+    }
 }

docs/docs/get_started/configuration.md

@@ -52,7 +52,7 @@ keys_path = "/path/to/keys"
 secrets_path = "/path/to.secrets"
 ```
 
-We currently support Lighthouse, Prysm, Teku and Lodestar's keystores so it's easier to load the keys. We're working on adding support for additional keystores. These are the expected file structures for each format:
+We currently support Lighthouse, Prysm, Teku, Lodestar, and Nimbus's keystores so it's easier to load the keys. We're working on adding support for additional keystores. These are the expected file structures for each format:
 
 <details>
   <summary>Lighthouse</summary>
@@ -175,6 +175,37 @@ We currently support Lighthouse, Prysm, Teku and Lodestar's keystores so it's ea
   :::
 </details>
 
+<details>
+  <summary>Nimbus</summary>
+
+  #### File structure:
+  ```
+  ├── keys
+  │   ├── <PUBLIC_KEY_1>
+  │   │   └── keystore.json
+  │   └── <PUBLIC_KEY_2>
+  │       └── keystore.json
+  └── secrets
+      ├── <PUBLIC_KEY_1>
+      └── <PUBLIC_KEY_2>
+  ```
+
+  #### Config:
+  ```toml
+  [pbs]
+  ...
+  with_signer = true
+
+  [signer]
+  port = 20000
+
+  [signer.local.loader]
+  format = "nimbus"
+  keys_path = "keys"
+  secrets_path = "secrets"
+  ```
+</details>
+
 ### Proxy keys store
 
 Proxy keys can be used to sign transactions with a different key than the one used to sign the block. Proxy keys are generated by the Signer module and authorized by the validator key. Each module have their own proxy keys, that can be BLS or ECDSA.