bugfix: handle new rate-limits for querying pools (#433)

Closes: #431 

* Added fast-near for testnet
* Rate limited querying pools to 140 / 30s

Ideally, if Fastnear is stable enough, the user shouldn't be able to hit
this limit as we will query information within the pools the user has
staked before.
But in case the fastener fails, we would need to query all active pools,
which number more than 150.

| Network | FastNear | Non fast near |
| :---:   | :---: | :---: |
| Testnet | < 140 (mostly 0-5) | 563 pools 
| Mainnet | < 140 ( mostly 0-5 ) | State of contract poolv1.near is too
large to be viewed |

@race-of-sloths

Author: akorchyn

src/commands/account/view_account_summary/mod.rs

@@ -108,8 +108,8 @@ fn get_account_inquiry(
                 crate::common::fetch_historically_delegated_staking_pools(fastnear_url, account_id)
                     .ok()
             });
-    let validators = if let Some(validators) = historically_delegated_validators {
-        Ok(validators)
+    let pools_to_query = if let Some(user_staked_pools) = historically_delegated_validators {
+        Ok(user_staked_pools)
     } else if let Some(staking_pools_factory_account_id) =
         &network_config.staking_pools_factory_account_id
     {
@@ -124,32 +124,69 @@ fn get_account_inquiry(
     let runtime = tokio::runtime::Builder::new_multi_thread()
         .enable_all()
         .build()?;
-    let concurrency = 10;
+    // Staring from Feb 2025, the rate limit is 150 requests per 30 seconds for mainnet.
+    // We will limit the number of requests per batch to 140 to be conservative.
+    let batch_size = 140;
+    let batch_cooldown = tokio::time::Duration::from_secs(30);
+    let concurrency = 10; // Process 10 requests concurrently within each batch
+
     let delegated_stake: color_eyre::Result<
         std::collections::BTreeMap<near_primitives::types::AccountId, near_token::NearToken>,
-    > = match validators {
-        Ok(validators) => Ok(runtime.block_on(
-            futures::stream::iter(validators)
-                .map(|validator_account_id| async {
-                    let balance = get_delegated_staked_balance(
-                        &json_rpc_client,
-                        block_reference,
-                        &validator_account_id,
-                        account_id,
-                    )
-                    .await?;
-                    Ok::<_, color_eyre::eyre::Report>((validator_account_id, balance))
-                })
-                .buffer_unordered(concurrency)
-                .filter(|balance_result| {
-                    futures::future::ready(if let Ok((_, balance)) = balance_result {
-                        !balance.is_zero()
-                    } else {
-                        true
-                    })
-                })
-                .try_collect(),
-        )?),
+    > = match pools_to_query {
+        Ok(validators) => {
+            let mut all_results = Ok(std::collections::BTreeMap::new());
+            let validators: Vec<_> = validators.into_iter().collect();
+
+            for (batch_index, validator_batch) in validators
+                .chunks(batch_size)
+                .map(|x| x.to_vec())
+                .enumerate()
+            {
+                if batch_index > 0 {
+                    // Wait 30 seconds before starting next batch
+                    tracing::info!(
+                        "Waiting for 30 seconds before fetching next batch of stake information"
+                    );
+                    runtime.block_on(async { tokio::time::sleep(batch_cooldown).await });
+                }
+
+                let batch_results = runtime.block_on(
+                    futures::stream::iter(validator_batch)
+                        .map(|validator_account_id| async {
+                            let balance = get_delegated_staked_balance(
+                                &json_rpc_client,
+                                block_reference,
+                                &validator_account_id,
+                                account_id,
+                            )
+                            .await?;
+                            Ok::<_, color_eyre::eyre::Report>((validator_account_id, balance))
+                        })
+                        .buffer_unordered(concurrency)
+                        .filter(|balance_result| {
+                            futures::future::ready(if let Ok((_, balance)) = balance_result {
+                                !balance.is_zero()
+                            } else {
+                                true
+                            })
+                        })
+                        .try_collect::<std::collections::BTreeMap<_, _>>(),
+                );
+
+                match batch_results {
+                    Ok(batch_results) => {
+                        let _ = all_results.as_mut().map(|all_results| {
+                            all_results.extend(batch_results);
+                        });
+                    }
+                    Err(err) => {
+                        all_results = Err(err);
+                        break;
+                    }
+                };
+            }
+            all_results
+        }
         Err(err) => Err(err),
     };
 

src/config/migrations.rs

@@ -1,4 +1,4 @@
-use crate::config::Config as ConfigV2;
+use crate::config::Config as ConfigV3;
 use crate::config::NetworkConfig as NetworkConfigV2;
 
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
@@ -7,6 +7,12 @@ pub struct ConfigV1 {
     pub network_connection: linked_hash_map::LinkedHashMap<String, NetworkConfigV1>,
 }
 
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct ConfigV2 {
+    pub credentials_home_dir: std::path::PathBuf,
+    pub network_connection: linked_hash_map::LinkedHashMap<String, NetworkConfigV2>,
+}
+
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub struct NetworkConfigV1 {
     pub network_name: String,
@@ -35,6 +41,25 @@ impl From<ConfigV1> for ConfigV2 {
     }
 }
 
+impl From<ConfigV2> for ConfigV3 {
+    fn from(config: ConfigV2) -> Self {
+        ConfigV3 {
+            credentials_home_dir: config.credentials_home_dir,
+            network_connection: config
+                .network_connection
+                .into_iter()
+                .map(|(network_name, mut network_config)| {
+                    if network_name == "testnet" && network_config.faucet_url.is_none() {
+                        network_config.fastnear_url =
+                            Some("https://test.api.fastnear.com/".parse().unwrap());
+                    }
+                    (network_name, network_config)
+                })
+                .collect(),
+        }
+    }
+}
+
 impl From<NetworkConfigV1> for NetworkConfigV2 {
     fn from(network_config: NetworkConfigV1) -> Self {
         match network_config.network_name.as_str() {
@@ -94,4 +119,7 @@ pub enum ConfigVersion {
     V1(ConfigV1),
     #[serde(rename = "2")]
     V2(ConfigV2),
+    // Adds fastnear_url to the testnet config if it's not present
+    #[serde(rename = "3")]
+    V3(ConfigV3),
 }

src/config/mod.rs

@@ -74,7 +74,7 @@ impl Config {
     }
 
     pub fn into_latest_version(self) -> migrations::ConfigVersion {
-        migrations::ConfigVersion::V2(self)
+        migrations::ConfigVersion::V3(self)
     }
 
     pub fn get_config_toml() -> color_eyre::eyre::Result<Self> {
@@ -193,7 +193,10 @@ impl From<migrations::ConfigVersion> for Config {
                     migrations::ConfigVersion::V2(config_v1.into())
                 }
                 migrations::ConfigVersion::V2(config_v2) => {
-                    break config_v2;
+                    migrations::ConfigVersion::V3(config_v2.into())
+                }
+                migrations::ConfigVersion::V3(config_v3) => {
+                    break config_v3;
                 }
             };
         }