A0-4231: Refactor all keystore and chainspec generation to seprate module (#1687)

# Description

Folow up after
5a8b1c3

This is 2nd out of 3 PRs to remove `aleph-runtime` dependency in
`aleph-node`. In this PR:
* The whole keystore and chainspec generation logic is moved to
`chain_spec` module. This module will be extracted as a whole, with
small adjustment, to a separate crate
* `bootstrap-chain` command no longer supports `--faucet-account`
switch. This is because only reason for it to exists was to set an
initial endowment, exactly what `--rich-accounts` does. So it's
suggested to migrate all `--faucet-accounts <account-id>` to
`--rich-accounts <account-id>`. This is a breaking change, yet no
impactful it's not used frequently.
* `bootstrap-node` command is removed. This is because it was doing the
same what `bootstrap-chain` does, which now have
`--authorities-account-ids` switch to differentiate between RPC and
validator nodes

## Type of change

Please delete options that are not relevant.

- New feature (non-breaking change which adds functionality)
- Breaking change (fix or feature that would cause existing
functionality to not work as expected)

# Checklist:

* e2e tests:
https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/8706331881

Author: Marcin-Radecki

.github/scripts/run_consensus.sh

@@ -74,20 +74,15 @@ function generate_account_ids() {
 
 function generate_chainspec() {
   local account_ids=("$@")
-
-  # First array element is RPC node, so not a validator
+  local all_account_ids=${account_ids[@]}
+  local account_ids_comma_separated="${all_account_ids//${IFS:0:1}/,}"
+  # the first array element is RPC node, so not a validator
   local validators=${account_ids[@]:1}
-  # comma separated ids
-  validator_ids="${validators//${IFS:0:1}/,}"
-
-  echo "Generate chainspec and keystores with sudo account //Alice for below validators..."
-  echo "${validator_ids}"
-  docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${NODE_IMAGE}" \
-  -c "aleph-node bootstrap-chain --base-path /data --account-ids "${validator_ids}" > /data/chainspec.json"
+  local validator_ids_comma_separated="${validators//${IFS:0:1}/,}"
 
-  echo "Generating keystore for RPC node ${account_ids[0]}..."
+  echo "Generate chainspec and keystores for accounts: ${account_ids_comma_separated[@]}"
   docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${NODE_IMAGE}" \
-  -c "aleph-node bootstrap-node --base-path /data/${account_ids[0]} --account-id ${account_ids[0]}" > /dev/null
+  -c "aleph-node bootstrap-chain --base-path /data --account-ids ${account_ids_comma_separated} --authorities-account-ids ${validator_ids_comma_separated}  > /data/chainspec.json"
 }
 
 function generate_bootnode_peer_id() {

aleph-client/src/connections.rs

@@ -419,7 +419,7 @@ impl<S: AsSigned + Sync> SignedConnectionApi for S {
 
 impl Connection {
     const DEFAULT_RETRIES: u32 = 10;
-    const RETRY_WAIT_SECS: u64 = 3;
+    const RETRY_WAIT_SECS: u64 = 6;
 
     /// Creates new connection from a given url.
     /// By default, it tries to connect 10 times, waiting 1 second between each unsuccessful attempt.

bin/node/src/chain_spec/builder.rs

@@ -1,20 +1,18 @@
 use std::string::ToString;
 
-use aleph_runtime::{Feature, Perbill, WASM_BINARY};
+use aleph_runtime::{Feature, WASM_BINARY};
 use pallet_staking::{Forcing, StakerStatus};
 use primitives::{
     staking::{MIN_NOMINATOR_BOND, MIN_VALIDATOR_BOND},
-    AccountId, AlephNodeSessionKeys as SessionKeys, Version as FinalityVersion, ADDRESSES_ENCODING,
+    AccountId, AlephNodeSessionKeys, Version as FinalityVersion, ADDRESSES_ENCODING,
     TOKEN_DECIMALS,
 };
 use serde_json::{Number, Value};
+use sp_runtime::Perbill;
 
-use crate::{
-    chain_spec::{cli::ChainParams, AlephNodeChainSpec},
-    commands::AuthorityKeys,
-};
+use crate::chain_spec::{cli::ChainSpecParams, keystore::AccountSessionKeys, AlephNodeChainSpec};
 
-fn to_account_ids(authorities: &[AuthorityKeys]) -> impl Iterator<Item = AccountId> + '_ {
+fn to_account_ids(authorities: &[AccountSessionKeys]) -> impl Iterator<Item = AccountId> + '_ {
     authorities.iter().map(|auth| auth.account_id.clone())
 }
 
@@ -37,13 +35,12 @@ fn system_properties(token_symbol: String) -> serde_json::map::Map<String, Value
 
 /// Generate chain spec for new AlephNode chains
 pub fn build_chain_spec(
-    chain_params: ChainParams,
-    authorities: Vec<AuthorityKeys>,
+    chain_params: &ChainSpecParams,
+    account_session_keys: Vec<AccountSessionKeys>,
 ) -> Result<AlephNodeChainSpec, String> {
     let token_symbol = String::from(chain_params.token_symbol());
     let sudo_account = chain_params.sudo_account_id();
     let rich_accounts = chain_params.rich_account_ids();
-    let faucet_account = chain_params.faucet_account_id();
     let finality_version = chain_params.finality_version();
 
     Ok(AlephNodeChainSpec::builder(
@@ -54,10 +51,9 @@ pub fn build_chain_spec(
     .with_id(chain_params.chain_id())
     .with_chain_type(chain_params.chain_type())
     .with_genesis_config_patch(generate_genesis_config(
-        authorities.clone(),    // Initial PoA authorities, will receive funds
-        sudo_account.clone(),   // Sudo account, will also be pre funded
-        rich_accounts.clone(),  // Pre-funded accounts
-        faucet_account.clone(), // Pre-funded faucet account
+        account_session_keys,
+        sudo_account,
+        rich_accounts,
         finality_version,
     ))
     .with_properties(system_properties(token_symbol))
@@ -67,58 +63,52 @@ pub fn build_chain_spec(
 /// Calculate initial endowments such that total issuance is kept approximately constant.
 fn calculate_initial_endowment(accounts: &[AccountId]) -> u128 {
     let total_issuance = 300_000_000u128 * 10u128.pow(TOKEN_DECIMALS);
-    // due to known issue https://github.com/paritytech/polkadot-sdk/pull/2987/files,
-    // we need to make sure returned number is un u64 range, otherwise serde_json::json macro fails
+    // (A0-4258) due to known issue https://github.com/paritytech/polkadot-sdk/pull/2987/files,
+    // we need to make sure returned number is in u64 range, otherwise serde_json::json macro fails
     // this is fixed in polkadot-sdk 1.6.0
     total_issuance / (accounts.len() as u128) / 10
 }
 
 /// Configure initial storage state for FRAME modules.
 fn generate_genesis_config(
-    authorities: Vec<AuthorityKeys>,
+    account_session_keys: Vec<AccountSessionKeys>,
     sudo_account: AccountId,
     rich_accounts: Option<Vec<AccountId>>,
-    faucet_account: Option<AccountId>,
     finality_version: FinalityVersion,
 ) -> serde_json::Value {
-    let mut endowed_accounts = to_account_ids(&authorities)
+    let mut endowed_accounts = to_account_ids(&account_session_keys)
         .chain(
             rich_accounts
                 .unwrap_or_default()
                 .into_iter()
                 .chain([sudo_account.clone()]),
         )
         .collect::<Vec<_>>();
-    if let Some(faucet_account) = faucet_account {
-        endowed_accounts.push(faucet_account);
-    }
     endowed_accounts.sort();
     endowed_accounts.dedup();
     let initial_endowement = calculate_initial_endowment(&endowed_accounts);
 
-    let initial_balances = endowed_accounts
-        .into_iter()
-        .map(|account| (account, initial_endowement))
-        .collect::<Vec<_>>();
-
     serde_json::json!({
         "balances": {
-            "balances": initial_balances,
+            "balances": endowed_accounts
+                        .into_iter()
+                        .map(|account| (account, initial_endowement))
+                        .collect::<Vec<_>>(),
         },
         "sudo": {
             "key": Some(sudo_account),
         },
         "elections": {
-            "reservedValidators": to_account_ids(&authorities).collect::<Vec<_>>(),
+            "reservedValidators": to_account_ids(&account_session_keys).collect::<Vec<_>>(),
         },
         "session": {
-           "keys": authorities
+           "keys": account_session_keys
                     .iter()
                     .map(|auth| {
                         (
                             auth.account_id.clone(),
                             auth.account_id.clone(),
-                            SessionKeys {
+                            AlephNodeSessionKeys {
                                 aura: auth.aura_key.clone(),
                                 aleph: auth.aleph_key.clone(),
                             },
@@ -128,10 +118,10 @@ fn generate_genesis_config(
         },
         "staking": {
             "forceEra": Forcing::NotForcing,
-            "validatorCount":  authorities.len() as u32,
+            "validatorCount":  account_session_keys.len() as u32,
             "minimumValidatorCount": 4,
             "slashRewardFraction": Perbill::from_percent(10),
-            "stakers": authorities
+            "stakers": account_session_keys
                         .iter()
                         .enumerate()
                         .map(|(validator_idx, validator)| {
@@ -153,11 +143,20 @@ fn generate_genesis_config(
         },
         "committeeManagement": {
             "sessionValidators": {
-                "committee": to_account_ids(&authorities).collect::<Vec<_>>(),
+                "committee": to_account_ids(&account_session_keys).collect::<Vec<_>>(),
             },
         },
         "featureControl": {
             "activeFeatures": vec![Feature::OnChainVerifier],
         },
     })
 }
+
+pub fn build_chain_spec_json(
+    is_raw_chainspec: bool,
+    chain_params: &ChainSpecParams,
+    account_session_keys: Vec<AccountSessionKeys>,
+) -> sc_service::error::Result<String> {
+    let chain_spec = build_chain_spec(chain_params, account_session_keys)?;
+    sc_service::chain_ops::build_spec(&chain_spec, is_raw_chainspec)
+}

bin/node/src/chain_spec/cli.rs

@@ -1,25 +1,13 @@
-use std::io::Write;
-
 use primitives::{AccountId, Version as FinalityVersion, LEGACY_FINALITY_VERSION};
 use sc_chain_spec::ChainType;
-use sc_cli::{
-    clap::{self, Args, Parser},
-    Error, KeystoreParams,
-};
-use sc_service::BasePath;
-use sp_application_crypto::Ss58Codec;
-
-use crate::{
-    chain_spec::{
-        builder::build_chain_spec, CHAINTYPE_DEV, CHAINTYPE_LIVE, CHAINTYPE_LOCAL,
-        DEFAULT_CHAIN_ID, DEFAULT_SUDO_ACCOUNT,
-    },
-    commands::{authority_keys, bootstrap_backup, open_keystore},
-    shared_params::SharedParams,
+use sc_cli::clap::{self, Args};
+
+use crate::chain_spec::{
+    parse_account_id, parse_chaintype, CHAINTYPE_LIVE, DEFAULT_CHAIN_ID, DEFAULT_SUDO_ACCOUNT_ALICE,
 };
 
 #[derive(Debug, Args, Clone)]
-pub struct ChainParams {
+pub struct ChainSpecParams {
     /// Chain ID is a short identifier of the chain
     #[arg(long, value_name = "ID", default_value = DEFAULT_CHAIN_ID)]
     chain_id: String,
@@ -36,28 +24,30 @@ pub struct ChainParams {
     #[arg(long, default_value = "DZERO")]
     token_symbol: String,
 
-    /// AccountIds of authorities forming the committee at the genesis (comma delimited)
+    /// all account ids that needs to have session keys generated when bootstraping chain (comma delimited)
     #[arg(long, value_delimiter = ',', value_parser = parse_account_id, num_args = 1..)]
     account_ids: Vec<AccountId>,
 
+    /// AccountIds of authorities forming the committee at the genesis (comma delimited)
+    /// If empty, then `--account-ids` are used as authorities.
+    /// If not empty, it should ba a subset of `--account-ids`.
+    #[arg(long, value_delimiter = ',', value_parser = parse_account_id, num_args = 1..)]
+    authorities_account_ids: Vec<AccountId>,
+
     /// AccountId of the sudo account
-    #[arg(long, value_parser = parse_account_id, default_value(DEFAULT_SUDO_ACCOUNT))]
+    #[arg(long, value_parser = parse_account_id, default_value(DEFAULT_SUDO_ACCOUNT_ALICE))]
     sudo_account_id: AccountId,
 
     /// Accounts that will receive initial endowment in genesis block
     #[arg(long, value_delimiter = ',', value_parser = parse_account_id, num_args = 1..)]
     rich_account_ids: Option<Vec<AccountId>>,
 
-    /// Optional faucet account to be endowed
-    #[arg(long, value_parser = parse_account_id)]
-    faucet_account_id: Option<AccountId>,
-
     /// Finality version at chain inception.
     #[arg(long, default_value = LEGACY_FINALITY_VERSION.to_string())]
     finality_version: FinalityVersion,
 }
 
-impl ChainParams {
+impl ChainSpecParams {
     pub fn chain_id(&self) -> &str {
         &self.chain_id
     }
@@ -78,6 +68,10 @@ impl ChainParams {
         self.account_ids.clone()
     }
 
+    pub fn authorities_account_ids(&self) -> Vec<AccountId> {
+        self.authorities_account_ids.clone()
+    }
+
     pub fn sudo_account_id(&self) -> AccountId {
         self.sudo_account_id.clone()
     }
@@ -86,81 +80,7 @@ impl ChainParams {
         self.rich_account_ids.clone()
     }
 
-    pub fn faucet_account_id(&self) -> Option<AccountId> {
-        self.faucet_account_id.clone()
-    }
-
     pub fn finality_version(&self) -> FinalityVersion {
         self.finality_version
     }
 }
-
-fn parse_chaintype(s: &str) -> Result<ChainType, Error> {
-    Ok(match s {
-        CHAINTYPE_DEV => ChainType::Development,
-        CHAINTYPE_LOCAL => ChainType::Local,
-        CHAINTYPE_LIVE => ChainType::Live,
-        s => panic!("Wrong chain type {s} Possible values: dev local live"),
-    })
-}
-
-/// The `bootstrap-chain` command is used to generate private keys for the genesis authorities
-/// keys are written to the keystore of the authorities
-/// and the chain specification is printed to stdout in the JSON format
-#[derive(Debug, Parser)]
-pub struct BootstrapChainCmd {
-    /// Force raw genesis storage output.
-    #[arg(long = "raw")]
-    pub raw: bool,
-
-    #[clap(flatten)]
-    pub keystore_params: KeystoreParams,
-
-    #[clap(flatten)]
-    pub chain_params: ChainParams,
-
-    #[clap(flatten)]
-    pub node_params: SharedParams,
-}
-
-/// Assumes an input path: some_path/, which is appended to finally become: some_path/account_id
-impl BootstrapChainCmd {
-    pub fn run(&self) -> Result<(), Error> {
-        let base_path = self.node_params.base_path();
-        let backup_dir = self.node_params.backup_dir();
-        let node_key_file = self.node_params.node_key_file();
-        let chain_id = self.chain_params.chain_id();
-
-        let genesis_authorities = self
-            .chain_params
-            .account_ids()
-            .into_iter()
-            .map(|account_id| {
-                let account_base_path: BasePath =
-                    base_path.path().join(account_id.to_string()).into();
-                bootstrap_backup(account_base_path.path(), backup_dir);
-                let keystore = open_keystore(&self.keystore_params, chain_id, &account_base_path);
-                authority_keys(
-                    &keystore,
-                    account_base_path.path(),
-                    node_key_file,
-                    account_id,
-                )
-            })
-            .collect();
-
-        let chain_spec = build_chain_spec(self.chain_params.clone(), genesis_authorities)?;
-
-        let json = sc_service::chain_ops::build_spec(&chain_spec, self.raw)?;
-        if std::io::stdout().write_all(json.as_bytes()).is_err() {
-            let _ = std::io::stderr().write_all(b"Error writing to stdout\n");
-        }
-
-        Ok(())
-    }
-}
-
-/// Generate AccountId based on string command line argument.
-fn parse_account_id(s: &str) -> Result<AccountId, Error> {
-    Ok(AccountId::from_string(s).expect("Passed string is not a hex encoding of a public key"))
-}