568 nodes list generation by pattern

CHANGELOG.md

@@ -5,6 +5,7 @@ Bob versions changelog
 ## [Unreleased]
 #### Added
 - Blob performes fsync if buffered bytes are larger than max_dirty_bytes_before_sync config param (#748)
+- Command to generate nodes section for  cluster.yaml or add new nodes to an existing cluster.yaml using range syntax patterns. (#568)
 
 #### Changed
 - Use cargo workspace to declare dependencies to avoid their duplication (#821)

bob-apps/Cargo.toml

@@ -43,6 +43,7 @@ env_logger = { workspace = true }
 clap = { workspace = true }
 tokio = { workspace = true, features = ["signal"] }
 pearl = { workspace = true }
+itertools = "0.10.5"
 
 
 [target.'cfg(all(target_env = "musl", target_arch = "x86_64", target_pointer_width = "64"))'.dependencies]

bob-apps/bin/ccg.rs

@@ -4,11 +4,13 @@ mod config_cluster_generator;
 extern crate log;
 
 use anyhow::{anyhow, Result as AnyResult};
-use bob::{ClusterConfig};
+use bob::ClusterConfig;
+use bob_common::configs::{cluster::DistributionFunc};
 use clap::{App, Arg, ArgMatches, SubCommand};
 use config_cluster_generator::{
     center::{check_expand_configs, get_new_disks, get_new_racks, Center},
-    utils::{init_logger, ceil, read_config_from_file, write_to_file},
+    pattern::{pattern_extend_nodes},
+    utils::{ceil, init_logger, read_config_from_file, write_to_file},
 };
 
 #[tokio::main]
@@ -28,6 +30,8 @@ fn try_main() -> AnyResult<()> {
     match get_matches().subcommand() {
         ("new", Some(matches)) => subcommand_new(matches),
         ("expand", Some(matches)) => subcommand_expand(matches),
+        ("new-hw", Some(matches)) => subcommand_new_hw(matches),
+        ("expand-hw", Some(matches)) => subcommand_expand_hw(matches),
         _ => Err(anyhow!("incorrect arguments: ERR")),
     }
 }
@@ -67,6 +71,39 @@ fn subcommand_expand(matches: &ArgMatches) -> AnyResult<()> {
     Ok(())
 }
 
+fn subcommand_new_hw(matches: &ArgMatches) -> AnyResult<()> {
+    debug!("start new config by pattern generation");
+    debug!("arguments: {:?}", matches);
+    let output = generate_hw_config(matches)?;
+    let output = serde_yaml::to_string(&output).expect("config serialization error");
+    debug!("config cluster generation: OK");
+    if let Some(name) = matches.value_of("output") {
+        write_to_file(output, name.to_owned());
+        debug!("output to file: OK");
+    } else {
+        println!("{}", output);
+        debug!("no file provided, stdout print: OK");
+    }
+    Ok(())
+}
+
+fn subcommand_expand_hw(matches: &ArgMatches) -> AnyResult<()> {
+    debug!("start config extending with new nodes by range pattern");
+    debug!("arguments: {:?}", matches);
+    let config = read_config_from_file(&get_input_config_name(matches))?;
+    let output = expand_hw_config(matches, config)?;
+    let output = serde_yaml::to_string(&output).expect("config serialization error");
+    debug!("config cluster extending: OK");
+    if let Some(name) = matches.value_of("output") {
+        write_to_file(output, name.to_owned());
+        debug!("output to file: OK");
+    } else {
+        println!("{}", output);
+        debug!("no file provided, stdout print: OK");
+    }
+    Ok(())
+}
+
 fn generate_config(matches: &ArgMatches, input: ClusterConfig) -> AnyResult<ClusterConfig> {
     let replicas_count = get_replicas_count(matches)?;
     let (total_vdisks, vdisks_per_disk) = get_vdisks_total_and_per_disk(matches)?;
@@ -93,6 +130,23 @@ fn expand_config(
     Ok(res)
 }
 
+fn generate_hw_config(matches: &ArgMatches) -> AnyResult<ClusterConfig> {
+    debug!("arguments: {:?}", matches);
+    let pattern = get_pattern(matches)?;
+    let node_pattern = get_nodename(matches);
+    let res = pattern_gen(pattern, node_pattern)?;
+    debug!("generate hw config: OK");
+    Ok(res)
+}
+
+fn expand_hw_config(matches: &ArgMatches, config: ClusterConfig) -> AnyResult<ClusterConfig> {
+    let pattern = get_pattern(matches)?;
+    let node_pattern = get_nodename(matches);
+    let res = pattern_expand(config, pattern, node_pattern)?;
+    debug!("expand hw config: OK");
+    Ok(res)
+}
+
 fn simple_expand(
     config: ClusterConfig,
     mut hardware_config: ClusterConfig,
@@ -153,6 +207,24 @@ fn simple_gen(
     Ok(config)
 }
 
+fn pattern_gen(pattern: String, node_pattern: String) -> AnyResult<ClusterConfig> {
+    let nodes = pattern_extend_nodes(vec![], pattern, node_pattern)?;
+    let config = ClusterConfig::new(nodes, vec![], vec![], DistributionFunc::default());
+    debug!("pattern gen: OK [\n{:#?}\n]", config);
+    Ok(config)
+}
+
+fn pattern_expand(
+    config: ClusterConfig,
+    pattern: String,
+    node_pattern: String,
+) -> AnyResult<ClusterConfig> {
+    let nodes = pattern_extend_nodes(config.nodes().to_owned(), pattern, node_pattern)?;
+    let config = ClusterConfig::new(nodes, config.vdisks().to_owned(), config.racks().to_owned(), config.distribution_func());
+    debug!("pattern extending: OK [\n{:#?}\n]", config);
+    Ok(config)
+}
+
 fn get_input_config_name(matches: &ArgMatches) -> String {
     let name = matches
         .value_of("input")
@@ -197,6 +269,24 @@ fn get_vdisks_total_and_per_disk(matches: &ArgMatches) -> AnyResult<(Option<usiz
     }
 }
 
+fn get_pattern(matches: &ArgMatches) -> AnyResult<String> {
+    if let Some(name) = matches.value_of("pattern") {
+        debug!("get_pattern: OK [{}]", name);
+        Ok(name.to_owned())
+    } else {
+        debug!("get_pattern: No value");
+        Err(anyhow!("Failed: no pattern present"))
+    }
+}
+
+fn get_nodename(matches: &ArgMatches) -> String {
+    let name = matches
+        .value_of("nodename")
+        .expect("is some, because of default arg value");
+    debug!("get_nodename: OK [{}]", name);
+    name.to_owned()
+}
+
 fn get_matches() -> ArgMatches<'static> {
     let input = Arg::with_name("input")
         .short("i")
@@ -231,6 +321,15 @@ fn get_matches() -> ArgMatches<'static> {
         .long("use-racks")
         .help("Use racks field in config")
         .takes_value(false);
+    let pattern_config = Arg::with_name("pattern")
+        .short("p")
+        .help("Pattern for pattern generation")
+        .takes_value(true);
+    let nodename_config = Arg::with_name("nodename")
+        .short("n")
+        .default_value("Node_{ip}_{port}_{id}")
+        .help("Node name pattern for pattern generation")
+        .takes_value(true);
     debug!("input arg: OK");
     let subcommand_expand = SubCommand::with_name("expand")
         .arg(input.clone())
@@ -239,15 +338,28 @@ fn get_matches() -> ArgMatches<'static> {
         .arg(hardware_config);
 
     let subcommand_new = SubCommand::with_name("new")
-        .arg(input)
-        .arg(output)
+        .arg(input.clone())
+        .arg(output.clone())
         .arg(vdisks_per_disk)
         .arg(vdisks_count)
         .arg(use_racks)
         .arg(replicas);
 
+    let subcommand_new_hw = SubCommand::with_name("new-hw")
+        .arg(output.clone())
+        .arg(pattern_config.clone())
+        .arg(nodename_config.clone());
+
+    let subcommand_expand_hw = SubCommand::with_name("expand-hw")
+        .arg(input.clone())
+        .arg(output.clone())
+        .arg(pattern_config)
+        .arg(nodename_config);
+
     App::new("Config Cluster Generator")
         .subcommand(subcommand_expand)
         .subcommand(subcommand_new)
+        .subcommand(subcommand_new_hw)
+        .subcommand(subcommand_expand_hw)
         .get_matches()
 }

bob-apps/bin/config_cluster_generator/mod.rs

@@ -1,2 +1,3 @@
 pub mod center;
+pub mod pattern;
 pub mod utils;