feat: added config checks (#310)

* feat: added checks for config variables

* refactor: used `validator` crate for config validation

* chore: simplified formatting

* chore: reordered imports

* feat: panic on error instead of logging

* chore: provided better message error

Author: frolvanya

Cargo.lock

@@ -11,7 +11,7 @@ redis_url = "${REDIS_URL}"
 [general.rpc_server]
 server_port = "${SERVER_PORT}"
 max_gas_burnt = "${MAX_GAS_BURNT}"
-contract_code_cache_size = "${CONTRACT_CODE_CAHCE_SIZE}"
+contract_code_cache_size = "${CONTRACT_CODE_CACHE_SIZE}"
 block_cache_size = "${BLOCK_CACHE_SIZE}"
 shadow_data_consistency_rate = "${SHADOW_DATA_CONSISTENCY_RATE}"
 prefetch_state_size_limit = "${PREFETCH_STATE_SIZE_LIMIT}"

config.toml

@@ -19,6 +19,7 @@ regex = "1.10.2"
 serde = "1.0.145"
 serde_derive = "1.0.145"
 serde_json = "1.0.108"
+validator = { version = "0.18.1", features = ["derive"] }
 opentelemetry = { version = "0.19", features = ["rt-tokio-current-thread"] }
 opentelemetry-jaeger = { version = "0.18", features = [
     "rt-tokio-current-thread",

configuration/Cargo.toml

@@ -1,14 +1,17 @@
-use crate::configs::deserialize_data_or_env;
 use near_lake_framework::near_indexer_primitives::near_primitives;
+use validator::Validate;
+
+use crate::configs::deserialize_data_or_env;
 
 // Database connection URL
 // Example: "postgres://user:password@localhost:5432/dbname"
 type DatabaseConnectUrl = String;
 
-#[derive(serde_derive::Deserialize, Debug, Clone, Default)]
+#[derive(Validate, serde_derive::Deserialize, Debug, Clone, Default)]
 pub struct ShardDatabaseConfig {
     #[serde(deserialize_with = "deserialize_data_or_env")]
     pub shard_id: u64,
+    #[validate(url(message = "Invalid database shard URL"))]
     #[serde(deserialize_with = "deserialize_data_or_env")]
     pub database_url: DatabaseConnectUrl,
 }
@@ -33,10 +36,12 @@ impl DatabaseConfig {
     }
 }
 
-#[derive(serde_derive::Deserialize, Debug, Clone, Default)]
+#[derive(Validate, serde_derive::Deserialize, Debug, Clone, Default)]
 pub struct CommonDatabaseConfig {
+    #[validate(url(message = "Invalid database URL"))]
     #[serde(deserialize_with = "deserialize_data_or_env")]
     pub database_url: DatabaseConnectUrl,
+    #[validate(nested)]
     #[serde(default)]
     pub shards: Vec<ShardDatabaseConfig>,
 }

configuration/src/configs/database.rs

@@ -1,6 +1,7 @@
 use std::str::FromStr;
 
 use serde_derive::Deserialize;
+use validator::Validate;
 
 use crate::configs::{
     deserialize_data_or_env, deserialize_optional_data_or_env, required_value_or_panic,
@@ -49,18 +50,23 @@ pub struct GeneralNearStateIndexerConfig {
     pub concurrency: usize,
 }
 
-#[derive(Deserialize, Debug, Clone, Default)]
+#[derive(Validate, Deserialize, Debug, Clone, Default)]
 pub struct CommonGeneralConfig {
     #[serde(deserialize_with = "deserialize_data_or_env")]
     pub chain_id: ChainId,
+    #[validate(url(message = "Invalid NEAR RPC URL"))]
     #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
     pub near_rpc_url: Option<String>,
+    #[validate(url(message = "Invalid NEAR Archival RPC URL"))]
     #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
     pub near_archival_rpc_url: Option<String>,
+    #[validate(url(message = "Invalid referer header value"))]
     #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
     pub referer_header_value: Option<String>,
+    #[validate(url(message = "Invalid Redis URL"))]
     #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
     pub redis_url: Option<String>,
+    #[validate(nested)]
     #[serde(default)]
     pub rpc_server: CommonGeneralRpcServerConfig,
     #[serde(default)]
@@ -95,16 +101,29 @@ impl FromStr for ChainId {
     }
 }
 
-#[derive(Deserialize, Debug, Clone)]
+#[derive(Validate, Deserialize, Debug, Clone)]
 pub struct CommonGeneralRpcServerConfig {
     #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
     pub server_port: Option<u16>,
     #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
     pub max_gas_burnt: Option<u64>,
+    #[validate(range(
+        min = 0.0,
+        message = "Contract code cache size must be greater than or equal to 0"
+    ))]
     #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
     pub contract_code_cache_size: Option<f64>,
+    #[validate(range(
+        min = 0.0,
+        message = "Block cache size must be greater than or equal to 0"
+    ))]
     #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
     pub block_cache_size: Option<f64>,
+    #[validate(range(
+        min = 0.0,
+        max = 100.0,
+        message = "Shadow data consistency rate must be between 0 and 100"
+    ))]
     #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
     pub shadow_data_consistency_rate: Option<f64>,
     #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]

configuration/src/configs/general.rs

@@ -4,6 +4,7 @@ use near_lake_framework::{
     near_indexer_primitives, near_indexer_primitives::views::StateChangeValueView,
 };
 use serde::Deserialize;
+use validator::Validate;
 
 pub(crate) mod database;
 pub(crate) mod general;
@@ -74,12 +75,14 @@ where
     })
 }
 
-#[derive(Deserialize, Debug, Clone, Default)]
+#[derive(Validate, Deserialize, Debug, Clone, Default)]
 pub struct CommonConfig {
+    #[validate(nested)]
     pub general: general::CommonGeneralConfig,
     #[serde(default)]
     pub rightsizing: rightsizing::CommonRightsizingConfig,
     pub lake_config: lake::CommonLakeConfig,
+    #[validate(nested)]
     pub database: database::CommonDatabaseConfig,
     // Set as default to avoid breaking changes
     // This options needs only for tx_indexer and rpc_server

configuration/src/configs/mod.rs

@@ -2,6 +2,8 @@ use std::path::PathBuf;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::util::SubscriberInitExt;
 
+use validator::Validate;
+
 mod configs;
 
 pub use crate::configs::database::DatabaseConfig;
@@ -18,6 +20,11 @@ where
     let path_root = find_configs_root().await?;
     load_env(path_root.clone()).await?;
     let common_config = read_toml_file(path_root).await?;
+
+    if let Err(validation_errors) = common_config.validate() {
+        panic!("Failed to validate config: {validation_errors}");
+    }
+
     Ok(T::from_common_config(common_config))
 }