change: ETCM-9687 small fixes and docs for db-sync data sources crate (#804)

Author: AmbientTea

changelog.md

@@ -5,6 +5,7 @@ This changelog is based on [Keep A Changelog](https://keepachangelog.com/en/1.1.
 # Unreleased
 
 ## Changed
+* `partner-chains-db-sync-data-sources` crate now exports all its public members from the root
 
 ## Removed
 

demo/node/src/data_sources.rs

@@ -1,10 +1,9 @@
 use authority_selection_inherents::authority_selection_inputs::AuthoritySelectionDataSource;
 use pallet_sidechain_rpc::SidechainRpcDataSource;
 use partner_chains_db_sync_data_sources::{
-	block::BlockDataSourceImpl, candidates::CandidatesDataSourceImpl,
-	governed_map::GovernedMapDataSourceCachedImpl, mc_hash::McHashDataSourceImpl,
-	metrics::McFollowerMetrics, native_token::NativeTokenManagementDataSourceImpl,
-	sidechain_rpc::SidechainRpcDataSourceImpl, stake_distribution::StakeDistributionDataSourceImpl,
+	BlockDataSourceImpl, CandidatesDataSourceImpl, GovernedMapDataSourceCachedImpl,
+	McFollowerMetrics, McHashDataSourceImpl, NativeTokenManagementDataSourceImpl,
+	SidechainRpcDataSourceImpl, StakeDistributionDataSourceImpl,
 };
 use partner_chains_mock_data_sources::{
 	block::BlockDataSourceMock, candidate::AuthoritySelectionDataSourceMock,
@@ -73,7 +72,7 @@ pub const GOVERNED_MAP_CACHE_SIZE: u16 = 100;
 pub async fn create_cached_db_sync_data_sources(
 	metrics_opt: Option<McFollowerMetrics>,
 ) -> Result<DataSources, Box<dyn Error + Send + Sync + 'static>> {
-	let pool = partner_chains_db_sync_data_sources::data_sources::get_connection_from_env().await?;
+	let pool = partner_chains_db_sync_data_sources::get_connection_from_env().await?;
 	// block data source is reused between mc_hash and sidechain_rpc to share cache
 	let block = Arc::new(BlockDataSourceImpl::new_from_env(pool.clone()).await?);
 	Ok(DataSources {

demo/node/src/service.rs

@@ -3,8 +3,8 @@
 use crate::data_sources::DataSources;
 use crate::inherent_data::{CreateInherentDataConfig, ProposalCIDP, VerifierCIDP};
 use crate::rpc::GrandpaDeps;
-use partner_chains_db_sync_data_sources::metrics::McFollowerMetrics;
-use partner_chains_db_sync_data_sources::metrics::register_metrics_warn_errors;
+use partner_chains_db_sync_data_sources::McFollowerMetrics;
+use partner_chains_db_sync_data_sources::register_metrics_warn_errors;
 use partner_chains_demo_runtime::{self, RuntimeApi, opaque::Block};
 use sc_client_api::BlockBackend;
 use sc_consensus_aura::{ImportQueueParams, SlotProportion, StartAuraParams};

toolkit/data-sources/cli/src/main.rs

@@ -1,10 +1,6 @@
 use authority_selection_inherents::authority_selection_inputs::AuthoritySelectionDataSource;
 use clap::Parser;
-use partner_chains_db_sync_data_sources::{
-	block::{BlockDataSourceImpl, DbSyncBlockDataSourceConfig},
-	candidates::CandidatesDataSourceImpl,
-	data_sources::{PgPool, read_mc_epoch_config},
-};
+use partner_chains_db_sync_data_sources::{BlockDataSourceImpl, CandidatesDataSourceImpl, PgPool};
 use sidechain_domain::*;
 use sp_timestamp::Timestamp;
 use std::error::Error;
@@ -86,7 +82,7 @@ mod data_source {
 	use super::*;
 
 	async fn pool() -> Result<PgPool> {
-		partner_chains_db_sync_data_sources::data_sources::get_connection_from_env().await
+		partner_chains_db_sync_data_sources::get_connection_from_env().await
 	}
 
 	pub struct BlockDataSourceWrapper {
@@ -118,11 +114,7 @@ mod data_source {
 
 	pub async fn block() -> Result<BlockDataSourceWrapper> {
 		Ok(BlockDataSourceWrapper {
-			inner: BlockDataSourceImpl::from_config(
-				pool().await?,
-				DbSyncBlockDataSourceConfig::from_env()?,
-				&read_mc_epoch_config()?,
-			),
+			inner: BlockDataSourceImpl::new_from_env(pool().await?).await?,
 		})
 	}
 

toolkit/data-sources/db-sync/src/block/mod.rs

@@ -1,3 +1,4 @@
+//! Db-Sync data source implementation that queries Cardano block information
 use crate::{
 	DataSourceError::*,
 	data_sources::read_mc_epoch_config,
@@ -20,22 +21,41 @@ use std::{
 #[cfg(test)]
 mod tests;
 
+/// Db-Sync data source that queries Cardano block information
+///
+/// This data source does not implement any data source interface used by one of the
+/// Partner Chain toolkit's features, but is used internally by other data sources
+/// that require access to Cardano block data
 #[allow(clippy::too_many_arguments)]
 #[derive(new)]
 pub struct BlockDataSourceImpl {
+	/// Postgres connection pool
 	pool: PgPool,
+	/// Cardano security parameter
+	///
+	/// This parameter controls how many confirmations (blocks on top) are required by
+	/// the Cardano node to consider a block to be stable. This is a network-wide parameter.
 	security_parameter: u32,
-	/// `security parameter / active slot coefficient` - minimal age of a block to be considered valid stable in relation to some given timestamp
+	/// Minimal age of a block to be considered valid stable in relation to some given timestamp.
+	/// Must be equal to `security parameter / active slot coefficient`.
 	min_slot_boundary_as_seconds: TimeDelta,
 	/// a characteristic of Ouroboros Praos and is equal to `3 * security parameter / active slot coefficient`
 	max_slot_boundary_as_seconds: TimeDelta,
+	/// Cardano main chain epoch configuration
 	mainchain_epoch_config: MainchainEpochConfig,
+	/// Additional offset applied when selecting the latest stable Cardano block
+	///
+	/// This parameter should be 0 by default and should only be increased to 1 in networks
+	/// struggling with frequent block rejections due to Db-Sync or Cardano node lag.
 	block_stability_margin: u32,
+	/// Number of contiguous Cardano blocks to be cached by this data source
 	cache_size: u16,
+	/// Internal block cache
 	stable_blocks_cache: Arc<Mutex<BlocksCache>>,
 }
 
 impl BlockDataSourceImpl {
+	/// Returns the latest _unstable_ Cardano block from the Db-Sync database
 	pub async fn get_latest_block_info(
 		&self,
 	) -> Result<MainchainBlock, Box<dyn std::error::Error + Send + Sync>> {
@@ -45,6 +65,9 @@ impl BlockDataSourceImpl {
 			.ok_or(ExpectedDataNotFound("No latest block on chain.".to_string()).into())
 	}
 
+	/// Returns the latest _stable_ Cardano block from the Db-Sync database that is within
+	/// acceptable bounds from `reference_timestamp`, accounting for the additional stability
+	/// offset configured by [block_stability_margin][Self::block_stability_margin].
 	pub async fn get_latest_stable_block_for(
 		&self,
 		reference_timestamp: Timestamp,
@@ -57,6 +80,8 @@ impl BlockDataSourceImpl {
 		Ok(block.map(From::from))
 	}
 
+	/// Finds a block by its `hash` and verifies that it is stable in reference to `reference_timestamp`
+	/// and returns its info
 	pub async fn get_stable_block_for(
 		&self,
 		hash: McBlockHash,
@@ -66,6 +91,7 @@ impl BlockDataSourceImpl {
 		self.get_stable_block_by_hash(hash, reference_timestamp).await
 	}
 
+	/// Finds a block by its `hash` and returns its info
 	pub async fn get_block_by_hash(
 		&self,
 		hash: McBlockHash,
@@ -83,15 +109,25 @@ impl BlockDataSourceImpl {
 	}
 }
 
+/// Configuration for [BlockDataSourceImpl]
 #[derive(Debug, Clone, Deserialize)]
 pub struct DbSyncBlockDataSourceConfig {
+	/// Cardano security parameter, ie. the number of confirmations needed to stabilize a block
 	pub cardano_security_parameter: u32,
-	//From shelley-genesis.json, example: "activeSlotsCoeff": 0.05,
+	/// Expected fraction of Cardano slots that will have a block produced
+	///
+	/// This value can be found in `shelley-genesis.json` file used by the Cardano node,
+	/// example: `"activeSlotsCoeff": 0.05`.
 	pub cardano_active_slots_coeff: f64,
+	/// Additional offset applied when selecting the latest stable Cardano block
+	///
+	/// This parameter should be 0 by default and should only be increased to 1 in networks
+	/// struggling with frequent block rejections due to Db-Sync or Cardano node lag.
 	pub block_stability_margin: u32,
 }
 
 impl DbSyncBlockDataSourceConfig {
+	/// Reads the config from environment
 	pub fn from_env() -> std::result::Result<Self, Box<dyn Error + Send + Sync + 'static>> {
 		let config: Self = Figment::new()
 			.merge(Env::raw())
@@ -103,6 +139,7 @@ impl DbSyncBlockDataSourceConfig {
 }
 
 impl BlockDataSourceImpl {
+	/// Creates a new instance of [BlockDataSourceImpl], reading configuration from the environment.
 	pub async fn new_from_env(
 		pool: PgPool,
 	) -> std::result::Result<Self, Box<dyn Error + Send + Sync + 'static>> {
@@ -113,6 +150,7 @@ impl BlockDataSourceImpl {
 		))
 	}
 
+	/// Creates a new instance of [BlockDataSourceImpl], using passed configuration.
 	pub fn from_config(
 		pool: PgPool,
 		DbSyncBlockDataSourceConfig {

toolkit/data-sources/db-sync/src/candidates/datum.rs

@@ -3,15 +3,15 @@ use partner_chains_plutus_data::permissioned_candidates::PermissionedCandidateDa
 use partner_chains_plutus_data::permissioned_candidates::PermissionedCandidateDatums;
 use sidechain_domain::*;
 
-pub fn raw_permissioned_candidate_data_from(
+pub(crate) fn raw_permissioned_candidate_data_from(
 	datum: PermissionedCandidateDatumV0,
 ) -> RawPermissionedCandidateData {
 	let PermissionedCandidateDatumV0 { sidechain_public_key, aura_public_key, grandpa_public_key } =
 		datum;
 	RawPermissionedCandidateData { sidechain_public_key, aura_public_key, grandpa_public_key }
 }
 
-pub fn raw_permissioned_candidate_data_vec_from(
+pub(crate) fn raw_permissioned_candidate_data_vec_from(
 	datums: PermissionedCandidateDatums,
 ) -> Vec<RawPermissionedCandidateData> {
 	match datums {

toolkit/data-sources/db-sync/src/candidates/mod.rs

@@ -1,3 +1,4 @@
+//! Db-Sync data source used by Partner Chain committee selection
 use crate::DataSourceError::*;
 use crate::db_model::{
 	self, Address, Asset, BlockNumber, EpochNumber, MainchainTxOutput, StakePoolEntry,
@@ -17,11 +18,11 @@ use sqlx::PgPool;
 use std::collections::HashMap;
 use std::error::Error;
 
-pub mod cached;
-pub mod datum;
+mod cached;
+mod datum;
 
 #[cfg(test)]
-pub mod tests;
+mod tests;
 
 #[derive(Clone, Debug)]
 struct ParsedCandidate {
@@ -45,8 +46,11 @@ struct RegisteredCandidate {
 	utxo_info: UtxoInfo,
 }
 
+/// Db-Sync data source serving data for Partner Chain committee selection
 pub struct CandidatesDataSourceImpl {
+	/// Postgres connection pool
 	pool: PgPool,
+	/// Prometheus metrics client
 	metrics_opt: Option<McFollowerMetrics>,
 }
 
@@ -121,6 +125,7 @@ impl AuthoritySelectionDataSource for CandidatesDataSourceImpl {
 });
 
 impl CandidatesDataSourceImpl {
+	/// Creates new instance of the data source
 	pub async fn new(
 		pool: PgPool,
 		metrics_opt: Option<McFollowerMetrics>,
@@ -130,6 +135,7 @@ impl CandidatesDataSourceImpl {
 		Ok(Self { pool, metrics_opt })
 	}
 
+	/// Creates a new caching instance of the data source
 	pub fn cached(
 		self,
 		candidates_for_epoch_cache_size: usize,

toolkit/data-sources/db-sync/src/data_sources.rs

@@ -1,5 +1,4 @@
-//! Data sources implementations that read from db-sync postgres.
-
+//! Helpers for configuring and creating a Postgres database connection
 use figment::Figment;
 use figment::providers::Env;
 use serde::Deserialize;
@@ -12,18 +11,24 @@ use std::fmt::Debug;
 use std::fmt::Formatter;
 use std::str::FromStr;
 
+/// Reads Cardano main chain epoch configuration from the environment.
+///
+/// See documentation of [MainchainEpochConfig::read_from_env] for the list of environment variables read.
 #[cfg(feature = "block-source")]
 pub fn read_mc_epoch_config() -> Result<MainchainEpochConfig, Box<dyn Error + Send + Sync>> {
 	Ok(MainchainEpochConfig::read_from_env()
 		.map_err(|e| format!("Failed to read main chain config: {}", e))?)
 }
 
+/// Postgres connection config used when creating a [PgPool].
 #[derive(Debug, Clone, Deserialize)]
 pub struct ConnectionConfig {
+	/// Postgres connection pool, eg. `postgres://postgres-user:postgres-password@db-sync-postgres-host:5432/db-sync-db`
 	pub(crate) db_sync_postgres_connection_string: SecretString,
 }
 
 impl ConnectionConfig {
+	/// Reads Postgres connection config from the environment
 	pub fn from_env() -> Result<Self, Box<dyn Error + Send + Sync + 'static>> {
 		let config: Self = Figment::new()
 			.merge(Env::raw())
@@ -42,7 +47,7 @@ impl Debug for SecretString {
 	}
 }
 
-pub async fn get_connection(
+async fn get_connection(
 	connection_string: &str,
 	acquire_timeout: std::time::Duration,
 ) -> Result<PgPool, Box<dyn Error + Send + Sync + 'static>> {
@@ -68,6 +73,11 @@ pub async fn get_connection(
 #[error("Could not connect to database: postgres://***:***@{0}:{1}/{2}; error: {3}")]
 struct PostgresConnectionError(String, u16, String, String);
 
+/// Returns a Postgres connection pool constructed using configuration read from environment
+///
+/// # Environment variables read:
+/// - `DB_SYNC_POSTGRES_CONNECTION_STRING`: Postgres connection pool, eg.
+///   `postgres://postgres-user:postgres-password@db-sync-postgres-host:5432/db-sync-db`
 pub async fn get_connection_from_env() -> Result<PgPool, Box<dyn Error + Send + Sync + 'static>> {
 	let config = ConnectionConfig::from_env()?;
 	get_connection(

toolkit/data-sources/db-sync/src/governed_map/mod.rs

@@ -1,3 +1,4 @@
+//! Db-Sync data source used by Partner Chain Governed Map feature
 use crate::DataSourceError::ExpectedDataNotFound;
 use crate::Result;
 use crate::block::BlockDataSourceImpl;
@@ -15,14 +16,20 @@ use std::cmp::{max, min};
 use std::sync::{Arc, Mutex};
 
 #[cfg(test)]
-pub mod tests;
+mod tests;
 
+/// Data source for the Governed Map feature of Partner Chains toolkit
+///
+/// See documentation of [sp_governed_map] for a description of the feature
 pub struct GovernedMapDataSourceImpl {
+	/// Postgres connection pool
 	pub pool: PgPool,
+	/// Prometheus metrics client
 	pub metrics_opt: Option<McFollowerMetrics>,
 }
 
 impl GovernedMapDataSourceImpl {
+	/// Creates a new instance of the data source
 	pub async fn new(
 		pool: PgPool,
 		metrics_opt: Option<McFollowerMetrics>,
@@ -105,15 +112,24 @@ async fn get_mappings_entries(
 	Ok(mappings)
 }
 
+/// Cached data source serving the Governed Map feature of Partner Chains toolkit
+///
+/// See documentation of [sp_governed_map] for a description of the feature
 pub struct GovernedMapDataSourceCachedImpl {
+	/// Postgres connection pool
 	pub pool: PgPool,
+	/// Prometheus metrics client
 	pub metrics_opt: Option<McFollowerMetrics>,
+	/// Internal data cache size
 	cache_size: u16,
+	/// Internal cache
 	cache: Arc<Mutex<Cache>>,
+	/// [BlockDataSourceImpl] instance shared with other data sources for cache reuse.
 	blocks: Arc<BlockDataSourceImpl>,
 }
 
 impl GovernedMapDataSourceCachedImpl {
+	/// Constructs a new Governed Map data source
 	pub async fn new(
 		pool: PgPool,
 		metrics_opt: Option<McFollowerMetrics>,