Add support to multiple proxies on net up --kubernetes

Author: ndr-ds

CLI.md

@@ -1102,6 +1102,9 @@ Start a Local Linera Network
   Default value: `1000000`
 * `--validators <VALIDATORS>` — The number of validators in the local test network
 
+  Default value: `1`
+* `--proxies <PROXIES>` — The number of proxies in the local test network
+
   Default value: `1`
 * `--shards <SHARDS>` — The number of shards per validator in the local test network
 

linera-service/src/cli/command.rs

@@ -1047,6 +1047,10 @@ pub enum NetCommand {
         #[arg(long, default_value = "1")]
         validators: usize,
 
+        /// The number of proxies in the local test network.
+        #[arg(long, default_value = "1")]
+        proxies: usize,
+
         /// The number of shards per validator in the local test network.
         #[arg(long, default_value = "1")]
         shards: usize,

linera-service/src/cli/main.rs

@@ -2215,6 +2215,7 @@ async fn run(options: &ClientOptions) -> Result<i32, Error> {
                 other_initial_chains,
                 initial_amount,
                 validators,
+                proxies,
                 shards,
                 testing_prng_seed,
                 policy_config,
@@ -2234,6 +2235,7 @@ async fn run(options: &ClientOptions) -> Result<i32, Error> {
                     *other_initial_chains,
                     *initial_amount,
                     *validators,
+                    *proxies,
                     *shards,
                     *testing_prng_seed,
                     binaries,

linera-service/src/cli/net_up_utils.rs

@@ -113,6 +113,7 @@ pub async fn handle_net_up_kubernetes(
     num_other_initial_chains: u32,
     initial_amount: u128,
     num_initial_validators: usize,
+    num_proxies: usize,
     num_shards: usize,
     testing_prng_seed: Option<u64>,
     binaries: &Option<Option<PathBuf>>,
@@ -130,6 +131,10 @@ pub async fn handle_net_up_kubernetes(
         num_initial_validators >= 1,
         "The local test network must have at least one validator."
     );
+    assert!(
+        num_proxies >= 1,
+        "The local test network must have at least one proxy."
+    );
     assert!(
         num_shards >= 1,
         "The local test network must have at least one shard per validator."
@@ -150,6 +155,7 @@ pub async fn handle_net_up_kubernetes(
         num_other_initial_chains,
         initial_amount: Amount::from_tokens(initial_amount),
         num_initial_validators,
+        num_proxies,
         num_shards,
         binaries: binaries.clone().into(),
         no_build,

linera-service/src/cli_wrappers/helmfile.rs

@@ -14,6 +14,7 @@ impl HelmFile {
     pub async fn sync(
         server_config_id: usize,
         github_root: &Path,
+        num_proxies: usize,
         num_shards: usize,
         cluster_id: u32,
         docker_image_name: String,
@@ -40,6 +41,7 @@ impl HelmFile {
                 "LINERA_HELMFILE_SET_SERVER_CONFIG",
                 format!("working/server_{server_config_id}.json"),
             )
+            .env("LINERA_HELMFILE_SET_NUM_PROXIES", num_proxies.to_string())
             .env("LINERA_HELMFILE_SET_NUM_SHARDS", num_shards.to_string())
             .env("LINERA_HELMFILE_LINERA_IMAGE", docker_image_name)
             .env(

linera-service/src/cli_wrappers/local_kubernetes_net.rs

@@ -62,6 +62,7 @@ pub struct LocalKubernetesNetConfig {
     pub num_other_initial_chains: u32,
     pub initial_amount: Amount,
     pub num_initial_validators: usize,
+    pub num_proxies: usize,
     pub num_shards: usize,
     pub binaries: BuildArg,
     pub no_build: bool,
@@ -90,6 +91,7 @@ pub struct LocalKubernetesNet {
     kubectl_instance: Arc<Mutex<KubectlInstance>>,
     kind_clusters: Vec<KindCluster>,
     num_initial_validators: usize,
+    num_proxies: usize,
     num_shards: usize,
     dual_store: bool,
 }
@@ -120,6 +122,7 @@ impl SharedLocalKubernetesNetTestingConfig {
             num_other_initial_chains: 2,
             initial_amount: Amount::from_tokens(2000),
             num_initial_validators: 4,
+            num_proxies: 1,
             num_shards: 4,
             binaries,
             no_build: false,
@@ -158,6 +161,7 @@ impl LineraNetConfig for LocalKubernetesNetConfig {
             KubectlInstance::new(Vec::new()),
             clusters,
             self.num_initial_validators,
+            self.num_proxies,
             self.num_shards,
             self.dual_store,
         )?;
@@ -338,6 +342,7 @@ impl LocalKubernetesNet {
         kubectl_instance: KubectlInstance,
         kind_clusters: Vec<KindCluster>,
         num_initial_validators: usize,
+        num_proxies: usize,
         num_shards: usize,
         dual_store: bool,
     ) -> Result<Self> {
@@ -353,6 +358,7 @@ impl LocalKubernetesNet {
             kubectl_instance: Arc::new(Mutex::new(kubectl_instance)),
             kind_clusters,
             num_initial_validators,
+            num_proxies,
             num_shards,
             dual_store,
         })
@@ -365,40 +371,47 @@ impl LocalKubernetesNet {
         Ok(command)
     }
 
-    fn configuration_string(&self, server_number: usize) -> Result<String> {
-        let n = server_number;
-        let path = self.tmp_dir.path().join(format!("validator_{n}.toml"));
-        let port = 19100 + server_number;
-        let internal_port = 20100;
+    fn configuration_string(&self, validator_number: usize) -> Result<String> {
+        let path = self
+            .tmp_dir
+            .path()
+            .join(format!("validator_{validator_number}.toml"));
+        let public_port = 19100;
+        let private_port = 20100;
         let metrics_port = 21100;
+        let protocol = self.network.toml();
+        let host = self.network.localhost();
         let mut content = format!(
             r#"
-                server_config_path = "server_{n}.json"
-                host = "127.0.0.1"
-                port = {port}
-                [external_protocol]
-                Grpc = "ClearText"
-                [internal_protocol]
-                Grpc = "ClearText"
-
-                [[proxies]]
-                host = "proxy-0.proxy-internal.default.svc.cluster.local"
-                public_port = {port}
-                private_port = {internal_port}
-                metrics_port = {metrics_port}
+                server_config_path = "server_{validator_number}.json"
+                host = "{host}"
+                port = {public_port}
+                external_protocol = {protocol}
+                internal_protocol = {protocol}
+
             "#
         );
 
-        for k in 0..self.num_shards {
-            let shard_port = 19100;
-            let shard_metrics_port = 21100;
+        for proxy_id in 0..self.num_proxies {
+            content.push_str(&format!(
+                r#"
+                    [[proxies]]
+                    host = "proxy-{proxy_id}.proxy-internal.default.svc.cluster.local"
+                    public_port = {public_port}
+                    private_port = {private_port}
+                    metrics_port = {metrics_port}
+                "#
+            ));
+        }
+
+        for shard_id in 0..self.num_shards {
             content.push_str(&format!(
                 r#"
 
                 [[shards]]
-                host = "shards-{k}.shards.default.svc.cluster.local"
-                port = {shard_port}
-                metrics_port = {shard_metrics_port}
+                host = "shards-{shard_id}.shards.default.svc.cluster.local"
+                port = {public_port}
+                metrics_port = {metrics_port}
                 "#
             ));
         }
@@ -419,8 +432,8 @@ impl LocalKubernetesNet {
             self.testing_prng_seed = Some(seed + 1);
         }
         command.arg("--validators");
-        for i in 0..self.num_initial_validators {
-            command.arg(&self.configuration_string(i)?);
+        for validator_number in 0..self.num_initial_validators {
+            command.arg(&self.configuration_string(validator_number)?);
         }
         command
             .args(["--committee", "committee.json"])
@@ -457,10 +470,11 @@ impl LocalKubernetesNet {
 
         let kubectl_instance_clone = self.kubectl_instance.clone();
         let tmp_dir_path_clone = self.tmp_dir.path().to_path_buf();
+        let num_proxies = self.num_proxies;
         let num_shards = self.num_shards;
 
         let mut validators_initialization_futures = Vec::new();
-        for (i, kind_cluster) in self.kind_clusters.iter().cloned().enumerate() {
+        for (validator_number, kind_cluster) in self.kind_clusters.iter().cloned().enumerate() {
             let base_dir = base_dir.clone();
             let github_root = github_root.clone();
 
@@ -473,15 +487,16 @@ impl LocalKubernetesNet {
                 let cluster_id = kind_cluster.id();
                 kind_cluster.load_docker_image(&docker_image_name).await?;
 
-                let server_config_filename = format!("server_{}.json", i);
+                let server_config_filename = format!("server_{}.json", validator_number);
                 fs_err::copy(
                     tmp_dir_path.join(&server_config_filename),
                     base_dir.join(&server_config_filename),
                 )?;
 
                 HelmFile::sync(
-                    i,
+                    validator_number,
                     &github_root,
+                    num_proxies,
                     num_shards,
                     cluster_id,
                     docker_image_name,
@@ -492,7 +507,7 @@ impl LocalKubernetesNet {
                 let mut kubectl_instance = kubectl_instance.lock().await;
                 let proxy_service = "svc/proxy";
 
-                let local_port = 19100 + i;
+                let local_port = 19100 + validator_number;
                 kubectl_instance.port_forward(
                     proxy_service,
                     &format!("{local_port}:19100"),