update docs + more cleanup

Author: ltitanb

crates/cli/src/docker_init.rs

@@ -29,14 +29,11 @@ use docker_compose_types::{
 };
 use eyre::Result;
 use indexmap::IndexMap;
-use serde::Serialize;
 
 /// Name of the docker compose file
 pub(super) const CB_COMPOSE_FILE: &str = "cb.docker-compose.yml";
 /// Name of the envs file
 pub(super) const CB_ENV_FILE: &str = ".cb.env";
-/// Name of the Prometheus targets file
-pub(super) const CB_TARGETS_FILE: &str = "targets.json";
 
 const SIGNER_NETWORK: &str = "signer_network";
 
@@ -134,10 +131,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
                             "{} has an exported port on {}",
                             module_cid, metrics_port
                         ));
-                        targets.push(PrometheusTargetConfig {
-                            targets: vec![format!("{module_cid}:{metrics_port}")],
-                            labels: PrometheusLabelsConfig { job: module_cid.clone() },
-                        });
+                        targets.push(format!("{host_endpoint}"));
                         let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64);
                         module_envs.insert(key, val);
 
@@ -209,10 +203,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
                             "{} has an exported port on {}",
                             module_cid, metrics_port
                         ));
-                        targets.push(PrometheusTargetConfig {
-                            targets: vec![format!("{module_cid}:{metrics_port}")],
-                            labels: PrometheusLabelsConfig { job: module_cid.clone() },
-                        });
+                        targets.push(format!("{host_endpoint}"));
                         let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64);
                         module_envs.insert(key, val);
 
@@ -272,10 +263,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
         let host_endpoint = SocketAddr::from((metrics_config.host, metrics_port));
         ports.push(format!("{}:{}", host_endpoint, metrics_port));
         warnings.push(format!("cb_pbs has an exported port on {}", metrics_port));
-        targets.push(PrometheusTargetConfig {
-            targets: vec![format!("cb_pbs:{metrics_port}")],
-            labels: PrometheusLabelsConfig { job: "pbs".to_owned() },
-        });
+        targets.push(format!("{host_endpoint}"));
         let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64);
         pbs_envs.insert(key, val);
 
@@ -347,10 +335,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
                     let host_endpoint = SocketAddr::from((metrics_config.host, metrics_port));
                     ports.push(format!("{}:{}", host_endpoint, metrics_port));
                     warnings.push(format!("cb_signer has an exported port on {}", metrics_port));
-                    targets.push(PrometheusTargetConfig {
-                        targets: vec![format!("cb_signer:{metrics_port}")],
-                        labels: PrometheusLabelsConfig { job: "signer".to_owned() },
-                    });
+                    targets.push(format!("{host_endpoint}"));
                     let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64);
                     signer_envs.insert(key, val);
                 }
@@ -475,10 +460,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
                     let host_endpoint = SocketAddr::from((metrics_config.host, metrics_port));
                     ports.push(format!("{}:{}", host_endpoint, metrics_port));
                     warnings.push(format!("cb_signer has an exported port on {}", metrics_port));
-                    targets.push(PrometheusTargetConfig {
-                        targets: vec![format!("cb_signer:{metrics_port}")],
-                        labels: PrometheusLabelsConfig { job: "signer".to_owned() },
-                    });
+                    targets.push(format!("{host_endpoint}"));
                     let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64);
                     signer_envs.insert(key, val);
                 }
@@ -581,9 +563,11 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
     let compose_path = Path::new(&output_dir).join(CB_COMPOSE_FILE);
     std::fs::write(&compose_path, compose_str)?;
     if !warnings.is_empty() {
+        println!();
         for exposed_port in warnings {
             println!("Warning: {}", exposed_port);
         }
+        println!()
     }
     // if file logging is enabled, warn about permissions
     if let Some(logs_config) = cb_config.logs {
@@ -592,20 +576,20 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
             "Warning: file logging is enabled, you may need to update permissions for the logs directory. e.g. with:\n\t`sudo chown -R 10001:10001 {}`",
             log_dir.display()
         );
+        println!()
     }
 
-    println!("Compose file written to: {:?}", compose_path);
+    println!("Docker Compose file written to: {:?}", compose_path);
 
     // write prometheus targets to file
     if !targets.is_empty() {
-        let targets_str = serde_json::to_string_pretty(&targets)?;
-        let targets_path = Path::new(&output_dir).join(CB_TARGETS_FILE);
-        std::fs::write(&targets_path, targets_str)?;
-        println!("Prometheus targets file written to: {:?}", targets_path);
+        let targets = targets.join(", ");
+        println!("Note: Make sure to add these targets for Prometheus to scrape: {targets}");
+        println!("Check out the docs on how to configure Prometheus/Grafana/cAdvisor: https://commit-boost.github.io/commit-boost-client/get_started/running/metrics");
     }
 
     if envs.is_empty() {
-        println!("Run with:\n\t`commit-boost-cli start --docker {:?}`", compose_path);
+        println!("Run with:\n\tdocker compose -f {:?} up -d", compose_path);
     } else {
         // write envs to .env file
         let envs_str = {
@@ -619,12 +603,13 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
         std::fs::write(&env_path, envs_str)?;
         println!("Env file written to: {:?}", env_path);
 
+        println!();
         println!(
-            "Run with:\n\t`docker compose --env-file {:?} -f {:?} up -d`",
+            "Run with:\n\tdocker compose --env-file {:?} -f {:?} up -d",
             env_path, compose_path
         );
         println!(
-            "Stop with:\n\t`docker compose --env-file {:?} -f {:?} down`",
+            "Stop with:\n\tdocker compose --env-file {:?} -f {:?} down",
             env_path, compose_path
         );
     }
@@ -655,18 +640,6 @@ fn get_env_uval(k: &str, v: u64) -> (String, Option<SingleValue>) {
 //     (k.into(), Some(SingleValue::Bool(v)))
 // }
 
-/// A prometheus target, use to dynamically add targets to the prometheus config
-#[derive(Debug, Serialize)]
-struct PrometheusTargetConfig {
-    targets: Vec<String>,
-    labels: PrometheusLabelsConfig,
-}
-
-#[derive(Debug, Serialize)]
-struct PrometheusLabelsConfig {
-    job: String,
-}
-
 fn get_log_volume(maybe_config: &Option<LogsSettings>, module_id: &str) -> Option<Volumes> {
     maybe_config.as_ref().map(|config| {
         let p = config.log_dir_path.join(module_id.to_lowercase());

docs/docs/developing/commit-module.md

@@ -8,7 +8,7 @@ While a module can be written in any language, we currently provide some utiliti
 
 In Rust, we provide utilities to load and run modules. Simply add to your `Cargo.toml`:
 ```toml
-commit-boost = { git = "https://github.com/Commit-Boost/commit-boost-client", rev = "..." }
+commit-boost = { git = "https://github.com/Commit-Boost/commit-boost-client", version = "..." }
 ```
 
 You will now be able to import the utils with:

docs/docs/get_started/configuration.md

@@ -20,7 +20,7 @@ port = 18550
 url = ""
 
 [metrics]
-prometheus_config = "./docker/prometheus.yml"
+use_metrics = true
 ```
 
 You can find a list of MEV-Boost Holesky relays [here](https://www.coincashew.com/coins/overview-eth/mev-boost/mev-relay-list#holesky-testnet-relays).
@@ -282,8 +282,12 @@ A full example of a config file with Dirk can be found [here](https://github.com
 ## Custom module
 We currently provide a test module that needs to be built locally. To build the module run:
 ```bash
-bash scripts/build_local_modules.sh
+just docker-build-test-modules
 ```
+:::note
+We use `just` as command runner. If you don't have it installed, either install it from https://github.com/casey/just or run the commands manually from the `justfile` at the root of the repo.
+:::
+
 This will create a Docker image called `test_da_commit` that periodically requests signatures from the validator, and a `test_builder_log` module that logs BuilderAPI events.
 
 The `cb-config.toml` file needs to be updated as follows:
@@ -301,7 +305,7 @@ keys_path = "/path/to/keys"
 secrets_path = "/path/to.secrets"
 
 [metrics]
-prometheus_config = "./docker/prometheus.yml"
+use_metrics = true
 
 [[modules]]
 id = "DA_COMMIT"

docs/docs/get_started/overview.md

@@ -55,13 +55,10 @@ cargo build --release --bin commit-boost-cli
 
 and the modules as Docker images:
 ```bash
-bash scripts/build_local_images.sh
+docker build -t commitboost_pbs_default . -f ./provisioning/pbs.Dockerfile
+docker build -t commitboost_signer . -f ./provisioning/signer.Dockerfile
 ```
 
-:::note
-If you require `sudo` access to run Docker, you will need `sudo` to run some of the Commit-Boost commands. This is because under the hood Commit-Boost invokes the Docker API. You can double check this by running `docker info` in a terminal. Consider adding your user to the docker group following [ Docker’s official post-installation steps](https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user)
-:::
-
 This will create two local images called `commitboost_pbs_default` and `commitboost_signer` for the Pbs and Signer module respectively. Make sure to use these images in the `docker_image` field in the `[pbs]` and `[signer]` sections of the `.toml` config file, respectively.
 
 ### Binaries

docs/docs/get_started/running/docker.md

@@ -3,7 +3,7 @@ description: Run Commit-Boost with Docker
 ---
 
 # Docker
-The Commit-Boost CLI will generate a dynamic `docker-compose.yml` file using the provided `.toml` config file. This is the recommended approach as Docker provides sandboxing of the containers from the rest of your system.
+The Commit-Boost CLI generates a dynamic `docker-compose.yml` file using the provided `.toml` config file. This is the recommended approach as Docker provides sandboxing of the containers from the rest of your system.
 
 ## Init
 
@@ -20,28 +20,24 @@ This will create up to three files:
 
 To start Commit-Boost run:
 ```bash
-commit-boost-cli start --docker cb.docker-compose.yml [--env .cb.env]
+docker compose --env-file ".cb.env" -f ".cb.docker-compose.yml" up -d
 ```
 
-This will run `docker compose up` with the correct envs, and start up the services including PBS, commit modules (if any), and metrics collection (if enabled).
+This will run the all the configured services, including PBS, signer and modules (if any).
 
 The MEV-Boost server will be exposed at `pbs.port` from the config, `18550` in our example. You'll need to point your CL/Validator client to this port to be able to source blocks from the builder market.
 
-If enabled, this will also start a Prometheus server on port `9090` and a Grafana instance on port `3000`. In Grafana, you will also find some preset dashboards already connected.
-
-
 ## Logs
-
-To check logs, run:
+To check the logs, run:
 ```bash
-commit-boost-cli logs
+docker compose --env-file ".cb.env" -f ".cb.docker-compose.yml" logs -f
 ```
 This will currently show all logs from the different services via the Docker logs interface. Logs are also optionally saved to file, depending on your `[logs]` configuration.
 
 ## Stop
 
 To stop all the services and cleanup, simply run:
 ```bash
-commit-boost-cli stop
+docker compose --env-file ".cb.env" -f ".cb.docker-compose.yml" down
 ```
 This will wind down all services and clear internal networks and file mounts.

docs/docs/get_started/running/metrics.md

@@ -0,0 +1,94 @@
+---
+description: Setup metrics collection
+---
+
+# Metrics
+
+Commit-Boost can be configured to collect metrics from the different services and expose them to be scraped from Prometheus.
+
+Make use to add the `[metrics]` section to your config file:
+
+```toml
+[metrics]
+use_metrics = true
+```
+If the section is missing, metrics collection will be disabled. If you generated the `docker-compose.yml` file with `commit-boost-cli`, metrics ports will be automatically configured, and a sample `target.json` file will be created. If you're running the binaries directly, you will need to set the correct environment variables, as described in the [previous section](/get_started/running/binary#common).
+
+## Example setup
+
+:::note
+The following examples assume you're running Prometheus/Grafana on the same machine as Commit-Boost. In general you should avoid this setup, and instead run them on a separate machine. cAdvisor should run in the same machine as the containers you want to monitor.
+:::
+
+
+### cAdvisor
+[cAdvisor](https://github.com/google/cadvisor) is a tool for collecting and reporting resource usage and performance characteristics of running containers.
+
+```yml title="cb.docker-compose.yml"
+cb_cadvisor:
+    image: gcr.io/cadvisor/cadvisor
+    container_name: cb_cadvisor
+    ports:
+    - 127.0.0.1:8080:8080
+    volumes:
+    - /var/run/docker.sock:/var/run/docker.sock:ro
+    - /sys:/sys:ro
+    - /var/lib/docker/:/var/lib/docker:ro
+```
+
+### Prometheus
+
+For more information on how to setup Prometheus, see the [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/getting_started/).
+
+```yml title="cb.docker-compose.yml"
+cb_prometheus:
+    image: prom/prometheus:v3.0.0
+    container_name: cb_prometheus
+    ports:
+    - 127.0.0.1:9090:9090
+    volumes:
+    - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    - prometheus-data:/prometheus
+```
+
+```yml title="prometheus.yml"
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: "commit-boost"
+    static_configs:
+      - targets: ["cb_da_commit:10000", "cb_pbs:10001", "cb_signer:10002", "cb_cadvisor:8080"]
+```
+
+### Grafana
+For more information on how to setup Grafana, see the [Grafana documentation](https://grafana.com/docs/grafana/latest/getting-started/).
+
+```yml title="cb.docker-compose.yml"
+cb_grafana:
+    image: grafana/grafana:11.3.1
+    container_name: cb_grafana
+    ports:
+    - 127.0.0.1:3000:3000
+    volumes:
+    - ./grafana/datasources:/etc/grafana/provisioning/datasources
+    - grafana-data:/var/lib/grafana
+```
+
+```yml title="datasources.yml"
+apiVersion: 1
+
+datasources:
+  - name: prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    orgId: 1
+    url: http://cb_prometheus:9090
+    isDefault: true
+    editable: true
+```
+
+Once Grafana is running, you can [import](https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/) the Commit-Boost dashboards from [here](https://github.com/Commit-Boost/commit-boost-client/tree/main/provisioning/grafana), making sure to select the correct `Prometheus` datasource.
+
+

docs/sidebars.js

@@ -39,6 +39,7 @@ const sidebars = {
           items: [
             'get_started/running/docker',
             'get_started/running/binary',
+            'get_started/running/metrics',
 
           ],
         },