optimizing Cardano and Ergo

Author: pragmaxim

chains/cardano/README.md

@@ -10,7 +10,13 @@ sudo apt-get install rustup
 
 ### Requirements
 
-With current [setup](src/model_v1.rs) you need at least 16 cores CPU and 32GB RAM to run both Cardano Node and Redbit without crashing.
+With current [setup](src/model_v1.rs) you need at least 16 cores CPU and 32GB RAM to run both Cardano Node (needs 24GB) and Redbit but remaining RAM 
+is used for page caching so it might swap or kernel with throttle our process. If you don't have enough RAM, set :
+
+```
+db_cache_size_gb = "low"
+min_entity_batch_size = 200_000
+```
 
 ### Usage
 

chains/cardano/src/model_v1.rs

@@ -28,33 +28,33 @@ pub struct Timestamp(pub u32);
 
 #[entity]
 pub struct Block {
-    #[pk]
+    #[pk(db_cache = 1)]
     pub height: Height,
     pub header: BlockHeader,
     pub transactions: Vec<Transaction>,
 }
 
 #[entity]
 pub struct BlockHeader {
-    #[fk(one2one)]
+    #[fk(one2one, db_cache = 1)]
     pub height: Height,
-    #[column(index)]
+    #[column(index, db_cache = 1)]
     pub hash: BlockHash,
-    #[column(index)]
+    #[column(index, db_cache = 1)]
     pub prev_hash: BlockHash,
-    #[column(range)]
+    #[column(range, db_cache = 1)]
     pub slot: Slot,
-    #[column(range)]
+    #[column(range, db_cache = 1)]
     pub timestamp: Timestamp,
     #[column(transient)]
     pub weight: Weight,
 }
 
 #[entity]
 pub struct Transaction {
-    #[fk(one2many, db_cache = 1)]
+    #[fk(one2many, db_cache = 2)]
     pub id: BlockPointer,
-    #[column(index, used, shards = 4, db_cache = 4, lru_cache = 12)]
+    #[column(index, used, db_cache = 20, lru_cache = 20)]
     pub hash: TxHash,
     pub utxos: Vec<Utxo>,
     #[write_from_using(input_refs, hash)] // implement custom write_from_using function, see hook.rs
@@ -71,9 +71,9 @@ pub struct Utxo {
     pub id: TransactionPointer,
     #[column(db_cache = 1)]
     pub amount: u64,
-    #[column(dictionary, shards = 4, db_cache = 4, lru_cache = 8)]
+    #[column(dictionary, db_cache = 20, lru_cache = 20)]
     pub script_hash: ScriptHash,
-    #[column(dictionary, shards = 4, db_cache = 4, lru_cache = 8)]
+    #[column(dictionary, db_cache = 20, lru_cache = 20)]
     pub address: Address,
     pub assets: Vec<Asset>,
 }
@@ -84,11 +84,11 @@ pub struct Asset {
     pub id: UtxoPointer,
     #[column(db_cache = 1)]
     pub amount: u64,
-    #[column(index, shards = 3, db_cache = 2)]
+    #[column(index, db_cache = 2)]
     pub asset_action: AssetAction,
-    #[column(dictionary, shards = 5, db_cache = 4, lru_cache = 6)]
+    #[column(dictionary, db_cache = 20, lru_cache = 20)]
     pub name: AssetName,
-    #[column(dictionary, shards = 5, db_cache = 4, lru_cache = 6)]
+    #[column(dictionary, db_cache = 20, lru_cache = 20)]
     pub policy_id: PolicyId,
 }
 

chains/ergo/README.md

@@ -10,7 +10,13 @@ sudo apt-get install rustup
 
 ### Requirements
 
-With current [setup](src/model_v1.rs) you need at least 8 cores CPU and 16GB RAM to run both Ergo Node and Redbit without crashing.
+With current [setup](src/model_v1.rs) you need at least 8 cores CPU and 12GB RAM to run both Ergo Node and Redbit but remaining RAM is used for page caching
+so it might swap or kernel with throttle our process. If you don't have enough RAM, set : 
+
+```
+db_cache_size_gb = "low"
+min_entity_batch_size = 200_000
+```
 
 ### Usage
 
@@ -20,8 +26,8 @@ export ERGO__API_KEY="foo"           # not needed for syncing
 cargo run --release 
 ```
 
-Indexing might crash especially on laptops with Node running locally and not being synced yet.
-In that case, set `fetching_parallelism = "low"` to not put the Node and Laptop under heavy pressure.
+Do not use more than `fetching_parallelism = "mild"` or sharding as Ergo node reading is quite CPU intensive and increasing
+fetching parallelism eats all runtime.
 
 ### Rest API
 

chains/ergo/config/ergo.toml

@@ -1,3 +1,3 @@
 api_host = "http://127.0.0.1:9053"
 api_key = "foo"
-fetching_parallelism = "high"   # low / mild / high
+fetching_parallelism = "mild"   # low / mild / high

chains/ergo/config/settings.toml

@@ -2,12 +2,12 @@
 name = "ergo"
 db_path = "/opt/.chain"
 enable = true
-node_sync_interval_s = 1        # 0 means only chatching up but no periodic sync
-fork_detection_heights = 100    # max 255, how many blocks from tip to check for forks
-db_cache_size_gb = "mild"       # off / tiny / low / mild / high / ultra
-min_entity_batch_size = 1_000_000   # Sum of : inputs + outputs
-non_durable_batches = 50        # how many batches to commit non-durably before forcing a durable commit
-processing_parallelism = "high" # off / tiny / low / mild / high / ultra
+node_sync_interval_s = 1         # 0 means only chatching up but no periodic sync
+fork_detection_heights = 100     # max 255, how many blocks from tip to check for forks
+db_cache_size_gb = "mild"        # off / tiny / low / mild / high / ultra
+min_entity_batch_size = 500_000  # Sum of : inputs + outputs
+non_durable_batches = 50         # how many batches to commit non-durably before forcing a durable commit
+processing_parallelism = "high"  # off / tiny / low / mild / high / ultra
 validation_from_height = 0
 
 [http]

chains/ergo/src/model_v1.rs

@@ -21,31 +21,31 @@ pub use chain::*;
 
 #[entity]
 pub struct Block {
-    #[pk]
+    #[pk(db_cache = 1)]
     pub height: Height,
     pub header: BlockHeader,
     pub transactions: Vec<Transaction>,
 }
 
 #[entity]
 pub struct BlockHeader {
-    #[fk(one2one)]
+    #[fk(one2one, db_cache = 1)]
     pub height: Height,
-    #[column(index)]
+    #[column(index, db_cache = 1)]
     pub hash: BlockHash,
-    #[column(index)]
+    #[column(index, db_cache = 1)]
     pub prev_hash: BlockHash,
-    #[column(range)]
+    #[column(range, db_cache = 1)]
     pub timestamp: Timestamp,
     #[column(transient)]
     pub weight: Weight,
 }
 
 #[entity]
 pub struct Transaction {
-    #[fk(one2many, db_cache = 1)]
+    #[fk(one2many, db_cache = 2)]
     pub id: BlockPointer,
-    #[column(index, db_cache = 2)]
+    #[column(index, db_cache = 4)]
     pub hash: TxHash,
     pub utxos: Vec<Utxo>,
     #[write_from_using(input_refs, utxos)] // implement custom write_from_using function, see hook.rs
@@ -62,13 +62,13 @@ pub struct Utxo {
     pub id: TransactionPointer,
     #[column(db_cache = 1)]
     pub amount: u64,
-    #[column(index, used, shards = 4, db_cache = 5)]
+    #[column(index, used, db_cache = 30)]
     pub box_id: BoxId,
-    #[column(dictionary, shards = 4, db_cache = 5)]
+    #[column(dictionary, lru_cache = 20, db_cache = 30)]
     pub address: Address,
-    #[column(dictionary, shards = 4, db_cache = 5)]
+    #[column(dictionary, lru_cache = 20, db_cache = 30)]
     pub tree: Tree,
-    #[column(dictionary, shards = 4, db_cache = 5)]
+    #[column(dictionary, lru_cache = 10, db_cache = 20)]
     pub tree_template: TreeTemplate,
     pub assets: Vec<Asset>,
 }
@@ -79,9 +79,9 @@ pub struct Asset {
     pub id: UtxoPointer,
     #[column(db_cache = 1)]
     pub amount: u64,
-    #[column(index, shards = 3, db_cache = 2)]
+    #[column(index, lru_cache = 10, db_cache = 10)]
     pub asset_action: AssetAction,
-    #[column(dictionary, shards = 4, db_cache = 2)]
+    #[column(dictionary, lru_cache = 10, db_cache = 10)]
     pub name: AssetName,
 }
 

redbit/src/storage/table_writer_api.rs

@@ -199,7 +199,8 @@ pub enum WriterCommand<K: CopyOwnedValue + Send + 'static, V: Key + Send + 'stat
     Remove(K, Sender<Result<bool, AppError>>),
     QueryAndWrite {
         last_shards: Option<usize>,
-        values: Vec<(usize, V)>, sink: Arc<dyn Fn(Option<usize>, Vec<(usize, Option<ValueOwned<K>>)>) -> Result<(), AppError> + Send + Sync + 'static>
+        values: Vec<(usize, V)>,
+        sink: Arc<dyn Fn(Option<usize>, Vec<(usize, Option<ValueOwned<K>>)>) -> Result<(), AppError> + Send + Sync + 'static>
     },
     Range(K, K, Sender<Result<Vec<(ValueBuf<K>, ValueBuf<V>)>, AppError>>),
     Flush(Sender<Result<TaskResult, AppError>>),              // commit current tx, stay alive (idle)