diff --git a/docker/Dockerfile b/docker/Dockerfile index b6dc1749ec53..b5e0f15c3aa9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,6 +11,7 @@ # `aarch64-unknown-linux-gnu`. # - `build_flag` is either "--release" or an empty string. # - `build_folder` is either "release" or "debug". +# - `build_features` is a comma-separated list of features to build the binaries with. # Stage 1 - Generate recipe file for dependencies @@ -24,12 +25,14 @@ ARG binaries= ARG copy=${binaries:+_copy} ARG build_flag=--release ARG build_folder=release +ARG build_features=scylladb,metrics FROM rust:1.74-slim-bookworm AS builder ARG git_commit ARG target ARG build_flag ARG build_folder +ARG build_features RUN apt-get update && apt-get install -y \ pkg-config \ @@ -69,7 +72,7 @@ RUN cargo build ${build_flag:+"$build_flag"} \ --bin linera \ --bin linera-proxy \ --bin linera-server \ - --features scylladb,metrics + --features $build_features RUN mv \ target/"$target"/"$build_folder"/linera \ diff --git a/docker/compose-server-entrypoint.sh b/docker/compose-server-entrypoint.sh index de247d4d7e50..fff164bb76d0 100755 --- a/docker/compose-server-entrypoint.sh +++ b/docker/compose-server-entrypoint.sh @@ -1,7 +1,9 @@ #!/bin/sh +storage=$1 + exec ./linera-server run \ - --storage scylladb:tcp:scylla:9042 \ + --storage $storage \ --server /config/server.json \ --shard 0 \ --genesis /config/genesis.json diff --git a/docker/compose-server-init.sh b/docker/compose-server-init.sh index b77204c76234..f984ebabe6fa 100755 --- a/docker/compose-server-init.sh +++ b/docker/compose-server-init.sh @@ -1,7 +1,9 @@ #!/bin/sh +storage=$1 + while true; do - ./linera storage check-existence --storage "scylladb:tcp:scylla:9042" + ./linera storage check-existence --storage $storage status=$? if [ $status -eq 0 ]; then @@ -10,7 +12,7 @@ while true; do elif [ $status -eq 1 ]; then echo "Database does not exist, attempting to initialize..." if ./linera-server initialize \ - --storage scylladb:tcp:scylla:9042 \ + --storage $storage \ --genesis /config/genesis.json; then echo "Initialization successful." exit 0 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 8c2e99e40e78..db0343cffb45 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -30,7 +30,7 @@ services: image: "${LINERA_IMAGE:-linera}" deploy: replicas: 4 - command: [ "./compose-server-entrypoint.sh" ] + command: [ "./compose-server-entrypoint.sh", "scylladb:tcp:scylla:9042" ] volumes: - .:/config labels: @@ -42,7 +42,7 @@ services: shard-init: image: "${LINERA_IMAGE:-linera}" container_name: shard-init - command: [ "./compose-server-init.sh" ] + command: [ "./compose-server-init.sh", "scylladb:tcp:scylla:9042" ] volumes: - .:/config depends_on: diff --git a/docker/server-entrypoint.sh b/docker/server-entrypoint.sh index c14d26d10945..3784ad54a1af 100644 --- a/docker/server-entrypoint.sh +++ b/docker/server-entrypoint.sh @@ -2,9 +2,10 @@ # Extract the ordinal number from the pod hostname ORDINAL="${HOSTNAME##*-}" +storage=$1 exec ./linera-server run \ - --storage scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042 \ + --storage $storage \ --server /config/server.json \ --shard $ORDINAL \ --genesis /config/genesis.json diff --git a/docker/server-init.sh b/docker/server-init.sh index b65f8ccaa915..f984ebabe6fa 100644 --- a/docker/server-init.sh +++ b/docker/server-init.sh @@ -1,7 +1,9 @@ #!/bin/sh +storage=$1 + while true; do - ./linera storage check-existence --storage "scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042" + ./linera storage check-existence --storage $storage status=$? if [ $status -eq 0 ]; then @@ -10,7 +12,7 @@ while true; do elif [ $status -eq 1 ]; then echo "Database does not exist, attempting to initialize..." if ./linera-server initialize \ - --storage scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042 \ + --storage $storage \ --genesis /config/genesis.json; then echo "Initialization successful." exit 0 diff --git a/kubernetes/linera-validator/scylla-manager.values.yaml b/kubernetes/linera-validator/scylla-manager.values.yaml index 20c1612970fe..9eee16b1054f 100644 --- a/kubernetes/linera-validator/scylla-manager.values.yaml +++ b/kubernetes/linera-validator/scylla-manager.values.yaml @@ -14,7 +14,7 @@ scylla: tag: 5.4.3 agentImage: tag: 3.2.8 - datacenter: manager-dc + datacenter: validator racks: - name: manager-rack members: 1 diff --git a/kubernetes/linera-validator/templates/shards.yaml b/kubernetes/linera-validator/templates/shards.yaml index 8b6124ec6dd1..5ee113f99d52 100644 --- a/kubernetes/linera-validator/templates/shards.yaml +++ b/kubernetes/linera-validator/templates/shards.yaml @@ -35,7 +35,7 @@ spec: - name: linera-server-initializer image: {{ .Values.lineraImage }} imagePullPolicy: {{ .Values.lineraImagePullPolicy }} - command: ["./server-init.sh"] + command: ["./server-init.sh", {{ .Values.storage | quote }}] env: - name: RUST_LOG value: {{ .Values.logLevel }} @@ -45,11 +45,15 @@ spec: - name: config mountPath: "/config" readOnly: true + {{- if .Values.dualStore }} + - name: linera-db + mountPath: "/linera.db" + {{- end }} containers: - name: linera-server image: {{ .Values.lineraImage }} imagePullPolicy: {{ .Values.lineraImagePullPolicy }} - command: ["./server-entrypoint.sh"] + command: ["./server-entrypoint.sh", {{ .Values.storage | quote }}] env: - name: RUST_LOG value: {{ .Values.logLevel }} @@ -57,6 +61,10 @@ spec: - name: config mountPath: "/config" readOnly: true + {{- if .Values.dualStore }} + - name: linera-db + mountPath: "/linera.db" + {{- end }} volumes: - name: config configMap: @@ -66,3 +74,13 @@ spec: path: server.json - key: genesisConfig path: genesis.json + {{- if .Values.dualStore }} + volumeClaimTemplates: + - metadata: + name: linera-db + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.rocksdbStorageSize }} + {{- end }} diff --git a/kubernetes/linera-validator/values-local.yaml.gotmpl b/kubernetes/linera-validator/values-local.yaml.gotmpl index 3562c0a94de9..873d6a1b9029 100644 --- a/kubernetes/linera-validator/values-local.yaml.gotmpl +++ b/kubernetes/linera-validator/values-local.yaml.gotmpl @@ -7,6 +7,10 @@ logLevel: "debug" proxyPort: 19100 metricsPort: 21100 numShards: {{ env "LINERA_HELMFILE_SET_NUM_SHARDS" | default 10 }} +# Size of the RocksDB storage per shard, IF using `DualStore`. Otherwise will be ignored. +rocksdbStorageSize: {{ env "LINERA_HELMFILE_SET_ROCKSDB_STORAGE_SIZE" | default "2Gi" }} +storage: {{ env "LINERA_HELMFILE_SET_STORAGE" | default "scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042" }} +dualStore: {{ env "LINERA_HELMFILE_SET_DUAL_STORE" | default "false" }} # Loki loki-stack: diff --git a/linera-service/src/cli_wrappers/docker.rs b/linera-service/src/cli_wrappers/docker.rs index b83356618d55..c541faed1829 100644 --- a/linera-service/src/cli_wrappers/docker.rs +++ b/linera-service/src/cli_wrappers/docker.rs @@ -24,6 +24,7 @@ impl DockerImage { binaries: &BuildArg, github_root: &PathBuf, build_mode: &BuildMode, + dual_store: bool, ) -> Result { let build_arg = match binaries { BuildArg::Directory(bin_path) => { @@ -73,6 +74,10 @@ impl DockerImage { } } + if dual_store { + command.args(["--build-arg", "build_features=rocksdb,scylladb,metrics"]); + } + #[cfg(not(with_testing))] command .args([ diff --git a/linera-service/src/cli_wrappers/helmfile.rs b/linera-service/src/cli_wrappers/helmfile.rs index 25078743ade9..d4cd1b41a118 100644 --- a/linera-service/src/cli_wrappers/helmfile.rs +++ b/linera-service/src/cli_wrappers/helmfile.rs @@ -17,14 +17,25 @@ impl HelmFile { num_shards: usize, cluster_id: u32, docker_image_name: String, + dual_store: bool, ) -> Result<()> { let chart_dir = format!("{}/kubernetes/linera-validator", github_root.display()); let temp_dir = tempfile::tempdir()?; fs_extra::copy_items(&[&chart_dir], temp_dir.path(), &CopyOptions::new())?; - Command::new("helmfile") - .current_dir(temp_dir.path().join("linera-validator")) + let mut command = Command::new("helmfile"); + command.current_dir(temp_dir.path().join("linera-validator")); + + if dual_store { + command.env( + "LINERA_HELMFILE_SET_STORAGE", + "dualrocksdbscylladb:/linera.db:spawn_blocking:tcp:scylla-client.scylla.svc.cluster.local:9042", + ); + command.env("LINERA_HELMFILE_SET_DUAL_STORE", "true"); + } + + command .env( "LINERA_HELMFILE_SET_SERVER_CONFIG", format!("working/server_{server_config_id}.json"), diff --git a/linera-service/src/cli_wrappers/local_kubernetes_net.rs b/linera-service/src/cli_wrappers/local_kubernetes_net.rs index c2f15f1ca282..8d489dce828f 100644 --- a/linera-service/src/cli_wrappers/local_kubernetes_net.rs +++ b/linera-service/src/cli_wrappers/local_kubernetes_net.rs @@ -71,6 +71,7 @@ pub struct LocalKubernetesNetConfig { pub docker_image_name: String, pub build_mode: BuildMode, pub policy_config: ResourceControlPolicyConfig, + pub dual_store: bool, } /// A wrapper of [`LocalKubernetesNetConfig`] to create a shared local Kubernetes network @@ -93,6 +94,7 @@ pub struct LocalKubernetesNet { kind_clusters: Vec, num_initial_validators: usize, num_shards: usize, + dual_store: bool, } #[cfg(with_testing)] @@ -127,6 +129,7 @@ impl SharedLocalKubernetesNetTestingConfig { docker_image_name: String::from("linera:latest"), build_mode: BuildMode::Release, policy_config: ResourceControlPolicyConfig::Testnet, + dual_store: false, }) } } @@ -159,6 +162,7 @@ impl LineraNetConfig for LocalKubernetesNetConfig { clusters, self.num_initial_validators, self.num_shards, + self.dual_store, )?; let client = net.make_client().await; @@ -338,6 +342,7 @@ impl LocalKubernetesNet { kind_clusters: Vec, num_initial_validators: usize, num_shards: usize, + dual_store: bool, ) -> Result { Ok(Self { network, @@ -352,6 +357,7 @@ impl LocalKubernetesNet { kind_clusters, num_initial_validators, num_shards, + dual_store, }) } @@ -433,6 +439,7 @@ impl LocalKubernetesNet { &self.binaries, &github_root, &self.build_mode, + self.dual_store, ) .await?; self.docker_image_name.clone() @@ -460,6 +467,7 @@ impl LocalKubernetesNet { let tmp_dir_path = tmp_dir_path_clone.clone(); let docker_image_name = docker_image_name.clone(); + let dual_store = self.dual_store; let future = async move { let cluster_id = kind_cluster.id(); kind_cluster.load_docker_image(&docker_image_name).await?; @@ -470,7 +478,15 @@ impl LocalKubernetesNet { base_dir.join(&server_config_filename), )?; - HelmFile::sync(i, &github_root, num_shards, cluster_id, docker_image_name).await?; + HelmFile::sync( + i, + &github_root, + num_shards, + cluster_id, + docker_image_name, + dual_store, + ) + .await?; let mut kubectl_instance = kubectl_instance.lock().await; let output = kubectl_instance.get_pods(cluster_id).await?; diff --git a/linera-service/src/linera/command.rs b/linera-service/src/linera/command.rs index fb689600ea8d..326902944b6b 100644 --- a/linera-service/src/linera/command.rs +++ b/linera-service/src/linera/command.rs @@ -991,6 +991,12 @@ pub enum NetCommand { /// The number of block exporters per validator in the local test network. Default is 0. #[arg(long, default_value = "0")] block_exporters: u32, + + /// Use dual store (rocksdb and scylladb) instead of just scylladb. This is exclusive for + /// kubernetes deployments. + #[cfg(feature = "kubernetes")] + #[arg(long, default_value = "false")] + dual_store: bool, }, /// Print a bash helper script to make `linera net up` easier to use. The script is diff --git a/linera-service/src/linera/main.rs b/linera-service/src/linera/main.rs index c34cd0ee1a11..5bf932063bde 100644 --- a/linera-service/src/linera/main.rs +++ b/linera-service/src/linera/main.rs @@ -2000,6 +2000,7 @@ async fn run(options: &ClientOptions) -> Result { faucet_chain, faucet_port, faucet_amount, + dual_store, .. } => { net_up_utils::handle_net_up_kubernetes( @@ -2017,6 +2018,7 @@ async fn run(options: &ClientOptions) -> Result { *faucet_chain, *faucet_port, *faucet_amount, + *dual_store, ) .boxed() .await?; diff --git a/linera-service/src/linera/net_up_utils.rs b/linera-service/src/linera/net_up_utils.rs index 325dd6954c43..c94d5009e230 100644 --- a/linera-service/src/linera/net_up_utils.rs +++ b/linera-service/src/linera/net_up_utils.rs @@ -119,6 +119,7 @@ pub async fn handle_net_up_kubernetes( faucet_chain: Option, faucet_port: NonZeroU16, faucet_amount: Amount, + dual_store: bool, ) -> anyhow::Result<()> { if num_initial_validators < 1 { panic!("The local test network must have at least one validator."); @@ -148,6 +149,7 @@ pub async fn handle_net_up_kubernetes( docker_image_name, build_mode, policy_config, + dual_store, }; let (mut net, client) = config.instantiate().await?; let faucet_service = print_messages_and_create_faucet(