diff --git a/CLI.md b/CLI.md index aee0c9bdd6d..3c610944a6d 100644 --- a/CLI.md +++ b/CLI.md @@ -1223,6 +1223,9 @@ Start a Local Linera Network * `--with-block-exporter` — Whether to start a block exporter for each validator Default value: `false` +* `--num-block-exporters ` — The number of block exporters to start + + Default value: `1` * `--exporter-address ` — The address of the block exporter Default value: `localhost` diff --git a/docker/Dockerfile.indexer-test b/docker/Dockerfile.indexer similarity index 80% rename from docker/Dockerfile.indexer-test rename to docker/Dockerfile.indexer index f9d8dc42954..f5e376eb885 100644 --- a/docker/Dockerfile.indexer-test +++ b/docker/Dockerfile.indexer @@ -25,7 +25,7 @@ ARG binaries= ARG copy=${binaries:+_copy} # ARG build_flag=--release ARG build_folder=debug -ARG build_features=metrics +ARG build_features=scylladb,metrics ARG rustflags="-C force-frame-pointers=yes" FROM rust:1.74-slim-bookworm AS builder @@ -63,7 +63,7 @@ COPY linera-persistent linera-persistent COPY linera-version linera-version COPY linera-views linera-views COPY linera-views-derive linera-views-derive -COPY linera-web linera-web +COPY web web COPY linera-witty linera-witty COPY linera-witty-macros linera-witty-macros COPY scripts scripts @@ -73,22 +73,11 @@ ENV GIT_COMMIT=${git_commit} ENV RUSTFLAGS=${rustflags} -# Build linera core binaries (no database features needed) -RUN cargo build ${build_flag:+"$build_flag"} \ - --bin linera \ - --bin linera-proxy \ - --bin linera-server \ - --features $build_features - -# Build storage service (memory storage, no database features) -RUN cargo build ${build_flag:+"$build_flag"} \ - -p linera-storage-service - -# Build block exporter with metrics feature only +# Build block exporter with scylladb and metrics features RUN cargo build ${build_flag:+"$build_flag"} \ -p linera-service \ --bin linera-exporter \ - --features metrics + --features $build_features # Build indexer binaries (no scylladb needed for testing) RUN cargo build ${build_flag:+"$build_flag"} \ @@ -97,10 +86,6 @@ RUN cargo build ${build_flag:+"$build_flag"} \ # Move binaries to avoid directory conflicts and clean up to save space RUN mv \ - target/"$build_folder"/linera \ - target/"$build_folder"/linera-proxy \ - target/"$build_folder"/linera-server \ - target/"$build_folder"/linera-storage-server \ target/"$build_folder"/linera-exporter \ target/"$build_folder"/linera-indexer-grpc \ ./ @@ -109,10 +94,6 @@ RUN mv \ FROM scratch AS builder_copy ARG binaries COPY \ - "$binaries"/linera \ - "$binaries"/linera-server \ - "$binaries"/linera-proxy \ - "$binaries"/linera-storage-server \ "$binaries"/linera-exporter \ "$binaries"/linera-indexer-grpc \ ./ @@ -140,12 +121,8 @@ RUN update-ca-certificates ARG target -# Copy pre-built binaries for non-indexer services +# Copy only indexer and exporter binaries COPY --from=binaries \ - linera \ - linera-proxy \ - linera-server \ - linera-storage-server \ linera-indexer-grpc \ linera-exporter \ ./ diff --git a/kubernetes/linera-validator/exporter-config.toml.tpl b/kubernetes/linera-validator/exporter-config.toml.tpl new file mode 100644 index 00000000000..97d533187ea --- /dev/null +++ b/kubernetes/linera-validator/exporter-config.toml.tpl @@ -0,0 +1,29 @@ +id = {{ .exporterId }} + +metrics_port = {{ .Values.blockExporter.metricsPort }} + +[service_config] +host = "0.0.0.0" +port = {{ .Values.blockExporter.port }} + +[destination_config] +committee_destination = true + +[[destination_config.destinations]] +file_name = "/data/linera-exporter.log" +kind = "Logging" + +[[destination_config.destinations]] +kind = "Indexer" +tls = "ClearText" +port = {{ .Values.indexer.port }} +endpoint = "linera-indexer" + +[limits] +persistence_period_ms = 299000 +work_queue_size = 256 +blob_cache_weight_mb = 1024 +blob_cache_items_capacity = 8192 +block_cache_weight_mb = 1024 +block_cache_items_capacity = 8192 +auxiliary_cache_size_mb = 1024 \ No newline at end of file diff --git a/kubernetes/linera-validator/templates/block-exporter.yaml b/kubernetes/linera-validator/templates/block-exporter.yaml new file mode 100644 index 00000000000..e0c67ddbf54 --- /dev/null +++ b/kubernetes/linera-validator/templates/block-exporter.yaml @@ -0,0 +1,139 @@ +{{- if .Values.blockExporter.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: block-exporter-config +data: +{{- range $i := until (int .Values.blockExporter.replicas) }} + exporter-config-{{ $i }}.toml: | +{{ tpl ($.Files.Get "exporter-config.toml.tpl") (dict "exporterId" $i "Values" $.Values) | indent 4 }} +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: linera-block-exporter + labels: + app: linera-block-exporter +spec: + clusterIP: None + ports: + - port: {{ .Values.blockExporter.port }} + name: http + - port: {{ .Values.blockExporter.metricsPort }} + name: metrics + selector: + app: linera-block-exporter +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: linera-block-exporter +spec: + serviceName: linera-block-exporter + replicas: {{ .Values.blockExporter.replicas }} + selector: + matchLabels: + app: linera-block-exporter + template: + metadata: + labels: + app: linera-block-exporter + spec: + {{- if eq .Values.environment "GCP" }} + nodeSelector: + workload: system + tolerations: + - key: system + value: "true" + effect: NoSchedule + {{- end }} + initContainers: + - name: config-selector + image: busybox + command: + - sh + - -c + - | + ORDINAL=$(echo $HOSTNAME | sed 's/.*-//') + cp /configmap/exporter-config-${ORDINAL}.toml /config/exporter-config.toml + volumeMounts: + - name: configmap + mountPath: /configmap + - name: config + mountPath: /config + containers: + - name: linera-block-exporter + image: {{ .Values.indexer.image }} + imagePullPolicy: {{ .Values.indexer.imagePullPolicy }} + command: + - "./linera-exporter" + - "--storage" + - "{{ .Values.storage }}" + - "--config-path" + - "/config/exporter-config.toml" + - "--metrics-port" + - "{{ .Values.blockExporter.metricsPort }}" + ports: + - containerPort: {{ .Values.blockExporter.port }} + name: http + - containerPort: {{ .Values.blockExporter.metricsPort }} + name: metrics + env: + - name: RUST_LOG + value: {{ .Values.blockExporter.logLevel }} + - name: RUST_BACKTRACE + value: "1" + volumeMounts: + - name: config + mountPath: /config + readOnly: true + - name: exporter-data + mountPath: /data + livenessProbe: + tcpSocket: + port: {{ .Values.blockExporter.port }} + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + tcpSocket: + port: {{ .Values.blockExporter.port }} + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: configmap + configMap: + name: block-exporter-config + - name: config + emptyDir: {} + - name: exporter-data + persistentVolumeClaim: + claimName: exporter-data +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: exporter-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.blockExporter.storageSize }} +{{- if .Values.blockExporter.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: linera-block-exporter + labels: + app: linera-block-exporter +spec: + selector: + matchLabels: + app: linera-block-exporter + endpoints: + - port: metrics + path: /metrics +{{- end }} +{{- end }} diff --git a/kubernetes/linera-validator/templates/explorer.yaml b/kubernetes/linera-validator/templates/explorer.yaml new file mode 100644 index 00000000000..65b78bf70a3 --- /dev/null +++ b/kubernetes/linera-validator/templates/explorer.yaml @@ -0,0 +1,118 @@ +{{- if .Values.explorer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: linera-explorer + labels: + app: linera-explorer +spec: + ports: + - port: {{ .Values.explorer.frontendPort }} + name: frontend + - port: {{ .Values.explorer.apiPort }} + name: api + selector: + app: linera-explorer +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: linera-explorer +spec: + replicas: 1 + selector: + matchLabels: + app: linera-explorer + template: + metadata: + labels: + app: linera-explorer + spec: + {{- if eq .Values.environment "GCP" }} + nodeSelector: + workload: system + tolerations: + - key: system + value: "true" + effect: NoSchedule + {{- end }} + containers: + - name: linera-explorer + image: {{ .Values.explorer.image }} + imagePullPolicy: {{ .Values.explorer.imagePullPolicy }} + ports: + - containerPort: {{ .Values.explorer.frontendPort }} + name: frontend + - containerPort: {{ .Values.explorer.apiPort }} + name: api + env: + - name: NODE_ENV + value: "production" + - name: DB_PATH + value: "/data/indexer.db" + - name: EXPLORER_FRONTEND_PORT + value: "{{ .Values.explorer.frontendPort }}" + - name: EXPLORER_API_PORT + value: "{{ .Values.explorer.apiPort }}" + - name: LOG_LEVEL + value: {{ .Values.explorer.logLevel }} + volumeMounts: + - name: indexer-data + mountPath: /data + readOnly: true + livenessProbe: + httpGet: + path: /api/health + port: {{ .Values.explorer.apiPort }} + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /api/health + port: {{ .Values.explorer.apiPort }} + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: indexer-data + persistentVolumeClaim: + claimName: indexer-data +{{- if .Values.explorer.ingress.enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: linera-explorer + labels: + app: linera-explorer + {{- with .Values.explorer.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.explorer.ingress.tls }} + tls: + {{- range .Values.explorer.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.explorer.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: linera-explorer + port: + number: {{ $.Values.explorer.frontendPort }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/kubernetes/linera-validator/templates/indexer.yaml b/kubernetes/linera-validator/templates/indexer.yaml new file mode 100644 index 00000000000..0121644dcb6 --- /dev/null +++ b/kubernetes/linera-validator/templates/indexer.yaml @@ -0,0 +1,83 @@ +{{- if .Values.indexer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: linera-indexer + labels: + app: linera-indexer +spec: + ports: + - port: {{ .Values.indexer.port }} + name: grpc + selector: + app: linera-indexer +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: linera-indexer +spec: + replicas: 1 + selector: + matchLabels: + app: linera-indexer + template: + metadata: + labels: + app: linera-indexer + spec: + {{- if eq .Values.environment "GCP" }} + nodeSelector: + workload: system + tolerations: + - key: system + value: "true" + effect: NoSchedule + {{- end }} + containers: + - name: linera-indexer + image: {{ .Values.indexer.image }} + imagePullPolicy: {{ .Values.indexer.imagePullPolicy }} + command: + - "./linera-indexer-grpc" + - "--port" + - "{{ .Values.indexer.port }}" + - "--database-path" + - "{{ .Values.indexer.databasePath }}" + ports: + - containerPort: {{ .Values.indexer.port }} + name: grpc + env: + - name: RUST_LOG + value: {{ .Values.indexer.logLevel }} + - name: RUST_BACKTRACE + value: "1" + volumeMounts: + - name: indexer-data + mountPath: /data + livenessProbe: + tcpSocket: + port: {{ .Values.indexer.port }} + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + tcpSocket: + port: {{ .Values.indexer.port }} + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: indexer-data + persistentVolumeClaim: + claimName: indexer-data +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: indexer-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.indexer.storageSize }} +{{- end }} diff --git a/kubernetes/linera-validator/values-local.yaml.gotmpl b/kubernetes/linera-validator/values-local.yaml.gotmpl index 90c376c4229..a3d43a80d4d 100644 --- a/kubernetes/linera-validator/values-local.yaml.gotmpl +++ b/kubernetes/linera-validator/values-local.yaml.gotmpl @@ -17,6 +17,41 @@ usingLocalSsd: {{ env "LINERA_HELMFILE_SET_USING_LOCAL_SSD" | default "false" }} gcpRun: {{ env "LINERA_HELMFILE_SET_GCP_RUN" | default "false" }} storageReplicationFactor: {{ env "LINERA_HELMFILE_SET_STORAGE_REPLICATION_FACTOR" | default 1 }} +# Explorer Services (Indexer, Block Exporter, Web Explorer) +blockExporter: + enabled: {{ env "LINERA_HELMFILE_SET_EXPLORER_ENABLED" | default "false" }} + replicas: {{ env "LINERA_HELMFILE_NUM_BLOCK_EXPORTERS" | default "1" }} + port: {{ env "LINERA_HELMFILE_BLOCK_EXPORTER_PORT" | default "8882" }} + metricsPort: 9091 + storageSize: 1Gi + logLevel: {{ env "LINERA_HELMFILE_SET_BLOCK_EXPORTER_LOG_LEVEL" | default "info" }} + serviceMonitor: + enabled: true + +indexer: + enabled: {{ env "LINERA_HELMFILE_SET_EXPLORER_ENABLED" | default "false" }} + image: {{ env "LINERA_HELMFILE_INDEXER_IMAGE" | default "linera-indexer:latest" }} + imagePullPolicy: Never + port: 8081 + databasePath: /data/indexer.db + storageSize: 2Gi + logLevel: {{ env "LINERA_HELMFILE_SET_INDEXER_LOG_LEVEL" | default "info" }} + +explorer: + enabled: {{ env "LINERA_HELMFILE_SET_EXPLORER_ENABLED" | default "false" }} + image: {{ env "LINERA_HELMFILE_EXPLORER_IMAGE" | default "linera-explorer:latest" }} + imagePullPolicy: Never + frontendPort: 3001 + apiPort: 3002 + logLevel: {{ env "LINERA_HELMFILE_SET_EXPLORER_LOG_LEVEL" | default "info" }} + ingress: + enabled: {{ env "LINERA_HELMFILE_SET_EXPLORER_ENABLED" | default "false" }} + hosts: + - host: linera-explorer.local + paths: + - path: / + pathType: Prefix + # Loki loki-stack: loki: diff --git a/linera-rpc/src/config.rs b/linera-rpc/src/config.rs index 338ad1abe43..4717670a0b4 100644 --- a/linera-rpc/src/config.rs +++ b/linera-rpc/src/config.rs @@ -304,6 +304,12 @@ pub struct ExporterServiceConfig { pub port: u16, } +impl ExporterServiceConfig { + pub fn new(host: String, port: u16) -> ExporterServiceConfig { + ExporterServiceConfig { host, port } + } +} + #[test] fn cross_chain_config_to_args() { let config = CrossChainConfig::default(); diff --git a/linera-service/src/cli/command.rs b/linera-service/src/cli/command.rs index 60958df9cb3..2d4c7e21c2a 100644 --- a/linera-service/src/cli/command.rs +++ b/linera-service/src/cli/command.rs @@ -1149,6 +1149,10 @@ pub enum NetCommand { #[arg(long, default_value = "false")] with_block_exporter: bool, + /// The number of block exporters to start. + #[arg(long, default_value = "1")] + num_block_exporters: usize, + /// The address of the block exporter. #[arg(long, default_value = "localhost")] exporter_address: String, @@ -1157,6 +1161,16 @@ pub enum NetCommand { #[arg(long, default_value = "8081")] exporter_port: NonZeroU16, + /// The name of the indexer docker image to use. + #[cfg(feature = "kubernetes")] + #[arg(long, default_value = "linera-indexer:latest")] + indexer_image_name: String, + + /// The name of the explorer docker image to use. + #[cfg(feature = "kubernetes")] + #[arg(long, default_value = "linera-explorer:latest")] + explorer_image_name: String, + /// Use dual store (rocksdb and scylladb) instead of just scylladb. This is exclusive for /// kubernetes deployments. #[cfg(feature = "kubernetes")] diff --git a/linera-service/src/cli/main.rs b/linera-service/src/cli/main.rs index 98ddb917449..b8f568c6719 100644 --- a/linera-service/src/cli/main.rs +++ b/linera-service/src/cli/main.rs @@ -2266,6 +2266,10 @@ async fn run(options: &ClientOptions) -> Result { faucet_chain, faucet_port, faucet_amount, + with_block_exporter, + num_block_exporters, + indexer_image_name, + explorer_image_name, dual_store, .. } => { @@ -2285,6 +2289,10 @@ async fn run(options: &ClientOptions) -> Result { *faucet_chain, *faucet_port, *faucet_amount, + *with_block_exporter, + *num_block_exporters, + indexer_image_name.clone(), + explorer_image_name.clone(), *dual_store, ) .boxed() diff --git a/linera-service/src/cli/net_up_utils.rs b/linera-service/src/cli/net_up_utils.rs index 871e0dbe84f..7de14803516 100644 --- a/linera-service/src/cli/net_up_utils.rs +++ b/linera-service/src/cli/net_up_utils.rs @@ -6,7 +6,7 @@ use std::{num::NonZeroU16, str::FromStr}; use colored::Colorize as _; use linera_base::{data_types::Amount, listen_for_shutdown_signals, time::Duration}; use linera_client::client_options::ResourceControlPolicyConfig; -use linera_rpc::config::{CrossChainConfig, ExporterServiceConfig}; +use linera_rpc::config::CrossChainConfig; #[cfg(feature = "storage-service")] use linera_storage_service::{ child::{StorageService, StorageServiceGuard}, @@ -125,6 +125,10 @@ pub async fn handle_net_up_kubernetes( faucet_chain: Option, faucet_port: NonZeroU16, faucet_amount: Amount, + with_block_exporter: bool, + num_block_exporters: usize, + indexer_image_name: String, + explorer_image_name: String, dual_store: bool, ) -> anyhow::Result<()> { assert!( @@ -149,6 +153,16 @@ pub async fn handle_net_up_kubernetes( let shutdown_notifier = CancellationToken::new(); tokio::spawn(listen_for_shutdown_signals(shutdown_notifier.clone())); + let num_block_exporters = if with_block_exporter { + assert!( + num_block_exporters > 0, + "If --with-block-exporter is provided, --num-block-exporters must be greater than 0" + ); + num_block_exporters + } else { + 0 + }; + let config = LocalKubernetesNetConfig { network: Network::Grpc, testing_prng_seed, @@ -162,6 +176,9 @@ pub async fn handle_net_up_kubernetes( docker_image_name, build_mode, policy_config, + num_block_exporters, + indexer_image_name, + explorer_image_name, dual_store, }; let (mut net, client) = config.instantiate().await?; @@ -223,15 +240,11 @@ pub async fn handle_net_up_service( let network = NetworkConfig { external, internal }; let path_provider = PathProvider::from_path_option(path)?; let num_proxies = 1; // Local networks currently support exactly 1 proxy. - let block_exporters = if with_block_exporter { - let exporter_config = ExporterServiceConfig { - host: block_exporter_address, - port: block_exporter_port.into(), - }; - ExportersSetup::Remote(vec![exporter_config]) - } else { - ExportersSetup::Local(vec![]) - }; + let block_exporters = ExportersSetup::new( + with_block_exporter, + block_exporter_address, + block_exporter_port, + ); let config = LocalNetConfig { network, database, diff --git a/linera-service/src/cli_wrappers/docker.rs b/linera-service/src/cli_wrappers/docker.rs index c541faed182..5a5168f4eaf 100644 --- a/linera-service/src/cli_wrappers/docker.rs +++ b/linera-service/src/cli_wrappers/docker.rs @@ -10,6 +10,22 @@ use tokio::process::Command; use crate::cli_wrappers::local_kubernetes_net::BuildMode; +pub enum Dockerfile { + Main, + Indexer, + Explorer, +} + +impl Dockerfile { + pub fn path(&self) -> &'static str { + match self { + Dockerfile::Main => "docker/Dockerfile", + Dockerfile::Indexer => "docker/Dockerfile.indexer", + Dockerfile::Explorer => "docker/Dockerfile.explorer", + } + } +} + pub struct DockerImage { name: String, } @@ -20,83 +36,91 @@ impl DockerImage { } pub async fn build( - name: &str, - binaries: &BuildArg, - github_root: &PathBuf, - build_mode: &BuildMode, + name: String, + binaries: BuildArg, + github_root: PathBuf, + build_mode: BuildMode, dual_store: bool, + dockerfile: Dockerfile, ) -> Result { - let build_arg = match binaries { - BuildArg::Directory(bin_path) => { - // Get the binaries from the specified path - let bin_path = - diff_paths(bin_path, github_root).context("Getting relative path failed")?; - let bin_path_str = bin_path.to_str().context("Getting str failed")?; - format!("binaries={bin_path_str}") - } - BuildArg::ParentDirectory => { - // Get the binaries from current_binary_parent - let parent_path = current_binary_parent() - .expect("Fetching current binaries path should not fail"); - let bin_path = - diff_paths(parent_path, github_root).context("Getting relative path failed")?; - let bin_path_str = bin_path.to_str().context("Getting str failed")?; - format!("binaries={bin_path_str}") - } - BuildArg::Build => { - // Build inside the Docker container - let arch = std::env::consts::ARCH; - // Translate architecture for Docker build arg - let docker_arch = match arch { - "arm" => "aarch", - _ => arch, - }; - format!("target={}-unknown-linux-gnu", docker_arch) - } - }; - let docker_image = Self { name: name.to_owned(), }; let mut command = Command::new("docker"); command - .current_dir(github_root) + .current_dir(github_root.clone()) .arg("build") - .args(["-f", "docker/Dockerfile"]) - .args(["--build-arg", &build_arg]); - - match build_mode { - // Release is the default, so no need to add any arguments - BuildMode::Release => {} - BuildMode::Debug => { - command.args(["--build-arg", "build_folder=debug"]); - command.args(["--build-arg", "build_flag="]); + .args(["-f", dockerfile.path()]); + + if let Dockerfile::Main = dockerfile { + let build_arg = match binaries { + BuildArg::Directory(bin_path) => { + // Get the binaries from the specified path + let bin_path = diff_paths(bin_path, github_root) + .context("Getting relative path failed")?; + let bin_path_str = bin_path.to_str().context("Getting str failed")?; + format!("binaries={bin_path_str}") + } + BuildArg::ParentDirectory => { + // Get the binaries from current_binary_parent + let parent_path = current_binary_parent() + .expect("Fetching current binaries path should not fail"); + let bin_path = diff_paths(parent_path, github_root) + .context("Getting relative path failed")?; + let bin_path_str = bin_path.to_str().context("Getting str failed")?; + format!("binaries={bin_path_str}") + } + BuildArg::Build => { + // Build inside the Docker container + let arch = std::env::consts::ARCH; + // Translate architecture for Docker build arg + let docker_arch = match arch { + "arm" => "aarch", + _ => arch, + }; + format!("target={}-unknown-linux-gnu", docker_arch) + } + }; + + command.args(["--build-arg", &build_arg]); + + match build_mode { + // Release is the default, so no need to add any arguments + BuildMode::Release => {} + BuildMode::Debug => { + command.args(["--build-arg", "build_folder=debug"]); + command.args(["--build-arg", "build_flag="]); + } + } + + if dual_store { + command.args(["--build-arg", "build_features=rocksdb,scylladb,metrics"]); } - } - if dual_store { - command.args(["--build-arg", "build_features=rocksdb,scylladb,metrics"]); + #[cfg(not(with_testing))] + command + .args([ + "--build-arg", + &format!( + "git_commit={}", + linera_version::VersionInfo::get()?.git_commit + ), + ]) + .args([ + "--build-arg", + &format!( + "build_date={}", + // Same format as $(TZ=UTC date) + chrono::Utc::now().format("%a %b %d %T UTC %Y") + ), + ]); } - #[cfg(not(with_testing))] command - .args([ - "--build-arg", - &format!( - "git_commit={}", - linera_version::VersionInfo::get()?.git_commit - ), - ]) - .args([ - "--build-arg", - &format!( - "build_date={}", - // Same format as $(TZ=UTC date) - chrono::Utc::now().format("%a %b %d %T UTC %Y") - ), - ]); - - command.arg(".").args(["-t", name]).spawn_and_wait().await?; + .arg(".") + .args(["-t", &name]) + .spawn_and_wait() + .await?; Ok(docker_image) } } diff --git a/linera-service/src/cli_wrappers/helmfile.rs b/linera-service/src/cli_wrappers/helmfile.rs index 9afa3d2729f..b3b992236ee 100644 --- a/linera-service/src/cli_wrappers/helmfile.rs +++ b/linera-service/src/cli_wrappers/helmfile.rs @@ -8,9 +8,12 @@ use fs_extra::dir::CopyOptions; use linera_base::command::CommandExt; use tokio::process::Command; +pub const DEFAULT_BLOCK_EXPORTER_PORT: u16 = 8882; + pub struct HelmFile; impl HelmFile { + #[expect(clippy::too_many_arguments)] pub async fn sync( server_config_id: usize, github_root: &Path, @@ -18,6 +21,10 @@ impl HelmFile { num_shards: usize, cluster_id: u32, docker_image_name: String, + with_block_exporter: bool, + num_block_exporters: usize, + indexer_image_name: String, + explorer_image_name: String, dual_store: bool, ) -> Result<()> { let chart_dir = format!("{}/kubernetes/linera-validator", github_root.display()); @@ -36,6 +43,20 @@ impl HelmFile { command.env("LINERA_HELMFILE_SET_DUAL_STORE", "true"); } + if with_block_exporter { + command.env("LINERA_HELMFILE_SET_EXPLORER_ENABLED", "true"); + command.env( + "LINERA_HELMFILE_NUM_BLOCK_EXPORTERS", + num_block_exporters.to_string(), + ); + command.env( + "LINERA_HELMFILE_BLOCK_EXPORTER_PORT", + DEFAULT_BLOCK_EXPORTER_PORT.to_string(), + ); + command.env("LINERA_HELMFILE_INDEXER_IMAGE", indexer_image_name); + command.env("LINERA_HELMFILE_EXPLORER_IMAGE", explorer_image_name); + } + command .env( "LINERA_HELMFILE_SET_SERVER_CONFIG", diff --git a/linera-service/src/cli_wrappers/local_kubernetes_net.rs b/linera-service/src/cli_wrappers/local_kubernetes_net.rs index dee90401f93..eb4fc29240c 100644 --- a/linera-service/src/cli_wrappers/local_kubernetes_net.rs +++ b/linera-service/src/cli_wrappers/local_kubernetes_net.rs @@ -14,13 +14,13 @@ use linera_base::{ }; use linera_client::client_options::ResourceControlPolicyConfig; use tempfile::{tempdir, TempDir}; -use tokio::process::Command; +use tokio::{process::Command, task::JoinSet}; #[cfg(with_testing)] use {linera_base::command::current_binary_parent, tokio::sync::OnceCell}; use crate::cli_wrappers::{ - docker::{BuildArg, DockerImage}, - helmfile::HelmFile, + docker::{BuildArg, DockerImage, Dockerfile}, + helmfile::{HelmFile, DEFAULT_BLOCK_EXPORTER_PORT}, kind::KindCluster, kubectl::KubectlInstance, local_net::PathProvider, @@ -69,6 +69,9 @@ pub struct LocalKubernetesNetConfig { pub docker_image_name: String, pub build_mode: BuildMode, pub policy_config: ResourceControlPolicyConfig, + pub num_block_exporters: usize, + pub indexer_image_name: String, + pub explorer_image_name: String, pub dual_store: bool, } @@ -93,6 +96,9 @@ pub struct LocalKubernetesNet { num_initial_validators: usize, num_proxies: usize, num_shards: usize, + num_block_exporters: usize, + indexer_image_name: String, + explorer_image_name: String, dual_store: bool, } @@ -129,6 +135,9 @@ impl SharedLocalKubernetesNetTestingConfig { docker_image_name: String::from("linera:latest"), build_mode: BuildMode::Release, policy_config: ResourceControlPolicyConfig::Testnet, + num_block_exporters: 0, + indexer_image_name: String::from("linera-indexer:latest"), + explorer_image_name: String::from("linera-explorer:latest"), dual_store: false, }) } @@ -163,6 +172,9 @@ impl LineraNetConfig for LocalKubernetesNetConfig { self.num_initial_validators, self.num_proxies, self.num_shards, + self.num_block_exporters, + self.indexer_image_name, + self.explorer_image_name, self.dual_store, )?; @@ -344,6 +356,9 @@ impl LocalKubernetesNet { num_initial_validators: usize, num_proxies: usize, num_shards: usize, + num_block_exporters: usize, + indexer_image_name: String, + explorer_image_name: String, dual_store: bool, ) -> Result { Ok(Self { @@ -360,6 +375,9 @@ impl LocalKubernetesNet { num_initial_validators, num_proxies, num_shards, + num_block_exporters, + indexer_image_name, + explorer_image_name, dual_store, }) } @@ -415,6 +433,25 @@ impl LocalKubernetesNet { "# )); } + + if self.num_block_exporters > 0 { + for exporter_num in 0..self.num_block_exporters { + let block_exporter_port = DEFAULT_BLOCK_EXPORTER_PORT; + let block_exporter_host = + format!("linera-block-exporter-{exporter_num}.linera-block-exporter"); + let config_content = format!( + r#" + + [[block_exporters]] + host = "{block_exporter_host}" + port = {block_exporter_port} + "# + ); + + content.push_str(&config_content); + } + } + fs_err::write(&path, content)?; path.into_os_string().into_string().map_err(|error| { anyhow!( @@ -444,19 +481,53 @@ impl LocalKubernetesNet { async fn run(&mut self) -> Result<()> { let github_root = get_github_root().await?; - // Build Docker image - let docker_image_name = if self.no_build { - self.docker_image_name.clone() + // Build Docker images + let (docker_image_name, indexer_image_name, explorer_image_name) = if self.no_build { + ( + self.docker_image_name.clone(), + self.indexer_image_name.clone(), + self.explorer_image_name.clone(), + ) } else { - DockerImage::build( - &self.docker_image_name, - &self.binaries, - &github_root, - &self.build_mode, + let mut join_set = JoinSet::new(); + join_set.spawn(DockerImage::build( + self.docker_image_name.clone(), + self.binaries.clone(), + github_root.clone(), + self.build_mode.clone(), self.dual_store, + Dockerfile::Main, + )); + if self.num_block_exporters > 0 { + join_set.spawn(DockerImage::build( + self.indexer_image_name.clone(), + self.binaries.clone(), + github_root.clone(), + self.build_mode.clone(), + self.dual_store, + Dockerfile::Indexer, + )); + join_set.spawn(DockerImage::build( + self.explorer_image_name.clone(), + self.binaries.clone(), + github_root.clone(), + self.build_mode.clone(), + self.dual_store, + Dockerfile::Explorer, + )); + } + + join_set + .join_all() + .await + .into_iter() + .collect::>>()?; + + ( + self.docker_image_name.clone(), + self.indexer_image_name.clone(), + self.explorer_image_name.clone(), ) - .await?; - self.docker_image_name.clone() }; let base_dir = github_root @@ -482,10 +553,15 @@ impl LocalKubernetesNet { let tmp_dir_path = tmp_dir_path_clone.clone(); let docker_image_name = docker_image_name.clone(); + let indexer_image_name = indexer_image_name.clone(); + let explorer_image_name = explorer_image_name.clone(); let dual_store = self.dual_store; + let num_block_exporters = self.num_block_exporters; let future = async move { let cluster_id = kind_cluster.id(); kind_cluster.load_docker_image(&docker_image_name).await?; + kind_cluster.load_docker_image(&indexer_image_name).await?; + kind_cluster.load_docker_image(&explorer_image_name).await?; let server_config_filename = format!("server_{}.json", validator_number); fs_err::copy( @@ -500,6 +576,10 @@ impl LocalKubernetesNet { num_shards, cluster_id, docker_image_name, + num_block_exporters > 0, + num_block_exporters, + indexer_image_name, + explorer_image_name, dual_store, ) .await?; diff --git a/linera-service/src/cli_wrappers/local_net.rs b/linera-service/src/cli_wrappers/local_net.rs index 48ebb721eeb..a804f033430 100644 --- a/linera-service/src/cli_wrappers/local_net.rs +++ b/linera-service/src/cli_wrappers/local_net.rs @@ -6,6 +6,7 @@ use std::sync::LazyLock; use std::{ collections::BTreeMap, env, + num::NonZeroU16, path::{Path, PathBuf}, sync::Arc, time::Duration, @@ -194,6 +195,7 @@ pub struct LocalNetConfig { } /// The setup for the block exporters. +#[derive(Clone, PartialEq)] pub enum ExportersSetup { // Block exporters are meant to be started and managed by the testing framework. Local(Vec), @@ -201,6 +203,22 @@ pub enum ExportersSetup { Remote(Vec), } +impl ExportersSetup { + pub fn new( + with_block_exporter: bool, + block_exporter_address: String, + block_exporter_port: NonZeroU16, + ) -> ExportersSetup { + if with_block_exporter { + let exporter_config = + ExporterServiceConfig::new(block_exporter_address, block_exporter_port.into()); + ExportersSetup::Remote(vec![exporter_config]) + } else { + ExportersSetup::Local(vec![]) + } + } +} + /// A set of Linera validators running locally as native processes. pub struct LocalNet { network: NetworkConfig, diff --git a/linera-service/src/config.rs b/linera-service/src/config.rs index 2e8b9fb07c2..a087e270345 100644 --- a/linera-service/src/config.rs +++ b/linera-service/src/config.rs @@ -10,7 +10,7 @@ use serde::{ }; /// The configuration file for the linera-exporter. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct BlockExporterConfig { /// Identity for the block exporter state. pub id: u32, @@ -39,7 +39,7 @@ impl BlockExporterConfig { } /// Configuration file for the exports. -#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)] pub struct DestinationConfig { /// The destination URIs to export to. pub destinations: Vec, @@ -248,7 +248,7 @@ impl<'de> Deserialize<'de> for Destination { } /// The configuration file to impose various limits /// on the resources used by the linera-exporter. -#[derive(Serialize, Deserialize, Debug, Clone, Copy)] +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq)] pub struct LimitsConfig { /// Time period in milliseconds between periodic persistence /// to the shared storage.