Skip to content

Commit b570301

Browse files
jpraynaudjpraynaud
authored
Merge pull request #1713 from input-output-hk/jpraynaud/1698-ctx-prover-cache-pool-merkle-maps
Add Block Range Merkle Map cache pool in Cardano Transactions prover
2 parents 37d0390 + be085ae commit b570301

File tree

26 files changed

+580
-87
lines changed

26 files changed

+580
-87
lines changed

‎.github/workflows/actions/deploy-terraform-infrastructure/action.yml‎

Lines changed: 43 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,9 @@ inputs:
104104
description: If set no error is returned in case of unparsable block and an error log is written instead. Will be ignored on (pre)production networks.
105105
required: false
106106
default: "false"
107+
mithril_aggregator_cardano_transactions_prover_cache_pool_size:
108+
description: Mithril aggregator cardano transactions prover cache pool size.
109+
required: true
107110
prometheus_auth_username:
108111
description: Prometheus metrics endpoint username.
109112
required: false
@@ -154,45 +157,46 @@ runs:
154157
working-directory: mithril-infra
155158
run: |
156159
cat > ./env.variables.tfvars << EOF
157-
environment_prefix = "${{ inputs.environment_prefix }}"
158-
cardano_network = "${{ inputs.cardano_network }}"
159-
cardano_image_id = "${{ inputs.cardano_node_version }}"
160-
cardano_image_registry = "${{ inputs.cardano_node_docker_registry }}"
161-
google_region = "${{ inputs.google_region }}"
162-
google_zone = "${{ inputs.google_zone }}"
163-
google_machine_type = "${{ inputs.google_machine_type }}"
164-
google_compute_instance_boot_disk_size = "${{ inputs.google_compute_instance_boot_disk_size }}"
165-
google_compute_instance_data_disk_size = "${{ inputs.google_compute_instance_data_disk_size }}"
166-
google_service_credentials_json_file = "./google-application-credentials.json"
167-
mithril_use_p2p_network = "${{ inputs.mithril_use_p2p_network }}"
168-
mithril_api_domain = "${{ inputs.mithril_api_domain }}"
169-
mithril_image_id = "${{ inputs.mithril_image_id }}"
170-
mithril_genesis_verification_key_url = "${{ inputs.mithril_genesis_verification_key_url }}"
171-
mithril_genesis_secret_key = "${{ inputs.mithril_genesis_secret_key }}"
172-
mithril_protocol_parameters = ${{ fromJSON(inputs.mithril_protocol_parameters) }}
173-
mithril_signers = ${{ fromJSON(inputs.mithril_signers) }}
174-
mithril_era_reader_adapter_type = "${{ inputs.mithril_era_reader_adapter_type }}"
175-
mithril_era_reader_address_url = "${{ inputs.mithril_era_reader_address_url }}"
176-
mithril_era_reader_verification_key_url = "${{ inputs.mithril_era_reader_verification_key_url }}"
177-
mithril_era_reader_secret_key = "${{ inputs.mithril_era_reader_secret_key }}"
178-
mithril_aggregator_signed_entity_types = "${{ inputs.mithril_aggregator_signed_entity_types }}"
179-
mithril_aggregator_cdn_cname = "${{ inputs.mithril_aggregator_cdn_cname }}"
180-
mithril_aggregator_snapshot_use_cdn_domain = ${{ inputs.mithril_aggregator_snapshot_use_cdn_domain }}
181-
mithril_aggregator_snapshot_compression_algorithm = "${{ inputs.mithril_aggregator_snapshot_compression_algorithm }}"
182-
mithril_aggregator_zstandard_parameters_level = "${{ inputs.mithril_aggregator_zstandard_parameters_level }}"
183-
mithril_aggregator_zstandard_parameters_workers = "${{ inputs.mithril_aggregator_zstandard_parameters_workers }}"
184-
mithril_aggregator_cexplorer_pools_url = "${{ inputs.mithril_aggregator_cexplorer_pools_url }}"
185-
mithril_aggregator_allow_unparsable_block = "${{ inputs.mithril_aggregator_allow_unparsable_block }}"
186-
prometheus_auth_username = "${{ inputs.prometheus_auth_username }}"
187-
prometheus_auth_password = "${{ inputs.prometheus_auth_password }}"
188-
prometheus_ingest_host = "${{ inputs.prometheus_ingest_host }}"
189-
prometheus_ingest_username = "${{ inputs.prometheus_ingest_username }}"
190-
prometheus_ingest_password = "${{ inputs.prometheus_ingest_password }}"
191-
loki_auth_username = "${{ inputs.loki_auth_username }}"
192-
loki_auth_password = "${{ inputs.loki_auth_password }}"
193-
loki_ingest_host = "${{ inputs.loki_ingest_host }}"
194-
loki_ingest_username = "${{ inputs.loki_ingest_username }}"
195-
loki_ingest_password = "${{ inputs.loki_ingest_password }}"
160+
environment_prefix = "${{ inputs.environment_prefix }}"
161+
cardano_network = "${{ inputs.cardano_network }}"
162+
cardano_image_id = "${{ inputs.cardano_node_version }}"
163+
cardano_image_registry = "${{ inputs.cardano_node_docker_registry }}"
164+
google_region = "${{ inputs.google_region }}"
165+
google_zone = "${{ inputs.google_zone }}"
166+
google_machine_type = "${{ inputs.google_machine_type }}"
167+
google_compute_instance_boot_disk_size = "${{ inputs.google_compute_instance_boot_disk_size }}"
168+
google_compute_instance_data_disk_size = "${{ inputs.google_compute_instance_data_disk_size }}"
169+
google_service_credentials_json_file = "./google-application-credentials.json"
170+
mithril_use_p2p_network = "${{ inputs.mithril_use_p2p_network }}"
171+
mithril_api_domain = "${{ inputs.mithril_api_domain }}"
172+
mithril_image_id = "${{ inputs.mithril_image_id }}"
173+
mithril_genesis_verification_key_url = "${{ inputs.mithril_genesis_verification_key_url }}"
174+
mithril_genesis_secret_key = "${{ inputs.mithril_genesis_secret_key }}"
175+
mithril_protocol_parameters = ${{ fromJSON(inputs.mithril_protocol_parameters) }}
176+
mithril_signers = ${{ fromJSON(inputs.mithril_signers) }}
177+
mithril_era_reader_adapter_type = "${{ inputs.mithril_era_reader_adapter_type }}"
178+
mithril_era_reader_address_url = "${{ inputs.mithril_era_reader_address_url }}"
179+
mithril_era_reader_verification_key_url = "${{ inputs.mithril_era_reader_verification_key_url }}"
180+
mithril_era_reader_secret_key = "${{ inputs.mithril_era_reader_secret_key }}"
181+
mithril_aggregator_signed_entity_types = "${{ inputs.mithril_aggregator_signed_entity_types }}"
182+
mithril_aggregator_cdn_cname = "${{ inputs.mithril_aggregator_cdn_cname }}"
183+
mithril_aggregator_snapshot_use_cdn_domain = ${{ inputs.mithril_aggregator_snapshot_use_cdn_domain }}
184+
mithril_aggregator_snapshot_compression_algorithm = "${{ inputs.mithril_aggregator_snapshot_compression_algorithm }}"
185+
mithril_aggregator_zstandard_parameters_level = "${{ inputs.mithril_aggregator_zstandard_parameters_level }}"
186+
mithril_aggregator_zstandard_parameters_workers = "${{ inputs.mithril_aggregator_zstandard_parameters_workers }}"
187+
mithril_aggregator_cexplorer_pools_url = "${{ inputs.mithril_aggregator_cexplorer_pools_url }}"
188+
mithril_aggregator_allow_unparsable_block = "${{ inputs.mithril_aggregator_allow_unparsable_block }}"
189+
mithril_aggregator_cardano_transactions_prover_cache_pool_size = "${{ inputs.mithril_aggregator_cardano_transactions_prover_cache_pool_size }}"
190+
prometheus_auth_username = "${{ inputs.prometheus_auth_username }}"
191+
prometheus_auth_password = "${{ inputs.prometheus_auth_password }}"
192+
prometheus_ingest_host = "${{ inputs.prometheus_ingest_host }}"
193+
prometheus_ingest_username = "${{ inputs.prometheus_ingest_username }}"
194+
prometheus_ingest_password = "${{ inputs.prometheus_ingest_password }}"
195+
loki_auth_username = "${{ inputs.loki_auth_username }}"
196+
loki_auth_password = "${{ inputs.loki_auth_password }}"
197+
loki_ingest_host = "${{ inputs.loki_ingest_host }}"
198+
loki_ingest_username = "${{ inputs.loki_ingest_username }}"
199+
loki_ingest_password = "${{ inputs.loki_ingest_password }}"
196200
EOF
197201
terraform fmt ./env.variables.tfvars
198202
cat ./env.variables.tfvars

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -666,6 +666,7 @@ jobs:
666666
mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
667667
mithril_aggregator_cexplorer_pools_url: ${{ vars.AGGREGATOR_CEXPLORER_POOLS_URL }}
668668
mithril_aggregator_allow_unparsable_block: ${{ vars.AGGREGATOR_ALLOW_UNPARSABLE_BLOCK }}
669+
mithril_aggregator_cardano_transactions_prover_cache_pool_size: ${{ vars.AGGREGATOR_CARDANO_TRANSACTIONS_PROVER_CACHE_POOL_SIZE }}
669670
prometheus_auth_username: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
670671
prometheus_auth_password: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
671672
prometheus_ingest_host: ${{ vars.PROMETHEUS_INGEST_HOST }}

‎.github/workflows/pre-release.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ jobs:
219219
mithril_aggregator_zstandard_parameters_level: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_LEVEL }}
220220
mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
221221
mithril_aggregator_cexplorer_pools_url: ${{ vars.AGGREGATOR_CEXPLORER_POOLS_URL }}
222+
mithril_aggregator_cardano_transactions_prover_cache_pool_size: ${{ vars.AGGREGATOR_CARDANO_TRANSACTIONS_PROVER_CACHE_POOL_SIZE }}
222223
prometheus_auth_username: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
223224
prometheus_auth_password: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
224225
prometheus_ingest_host: ${{ vars.PROMETHEUS_INGEST_HOST }}

‎.github/workflows/release.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ jobs:
156156
mithril_aggregator_zstandard_parameters_level: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_LEVEL }}
157157
mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
158158
mithril_aggregator_cexplorer_pools_url: ${{ vars.AGGREGATOR_CEXPLORER_POOLS_URL }}
159+
mithril_aggregator_cardano_transactions_prover_cache_pool_size: ${{ vars.AGGREGATOR_CARDANO_TRANSACTIONS_PROVER_CACHE_POOL_SIZE }}
159160
prometheus_auth_username: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
160161
prometheus_auth_password: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
161162
prometheus_ingest_host: ${{ vars.PROMETHEUS_INGEST_HOST }}

‎.github/workflows/test-deploy-network.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ jobs:
134134
mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
135135
mithril_aggregator_cexplorer_pools_url: ${{ vars.AGGREGATOR_CEXPLORER_POOLS_URL }}
136136
mithril_aggregator_allow_unparsable_block: ${{ vars.AGGREGATOR_ALLOW_UNPARSABLE_BLOCK }}
137+
mithril_aggregator_cardano_transactions_prover_cache_pool_size: ${{ vars.AGGREGATOR_CARDANO_TRANSACTIONS_PROVER_CACHE_POOL_SIZE }}
137138
prometheus_auth_username: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
138139
prometheus_auth_password: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
139140
prometheus_ingest_host: ${{ vars.PROMETHEUS_INGEST_HOST }}

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ As a minor extension, we have adopted a slightly different versioning convention
2121

2222
- Implement a Chain Reader which retrieves blocks from the Cardano chain with Pallas through the `chainsync` mini-protocol.
2323

24+
- Implement a Resource Pool and use it for caching Block Range Merkle maps used by the Cardano transactions prover and improving the throughput.
25+
2426
- Crates versions:
2527

2628
| Crate | Version |

‎Cargo.lock‎

Lines changed: 4 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎mithril-aggregator/Cargo.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "mithril-aggregator"
3-
version = "0.5.12"
3+
version = "0.5.13"
44
description = "A Mithril Aggregator server"
55
authors = { workspace = true }
66
edition = { workspace = true }
@@ -31,6 +31,7 @@ mithril-doc = { path = "../internal/mithril-doc" }
3131
mithril-persistence = { path = "../internal/mithril-persistence" }
3232
openssl = { version = "0.10.63", features = ["vendored"], optional = true }
3333
openssl-probe = { version = "0.1.5", optional = true }
34+
rayon = "1.10.0"
3435
reqwest = { version = "0.12.0", features = ["json"] }
3536
semver = "1.0.21"
3637
serde = { version = "1.0.196", features = ["derive"] }

‎mithril-aggregator/src/artifact_builder/cardano_transactions.rs‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
1+
use std::sync::Arc;
2+
13
use async_trait::async_trait;
24

5+
use crate::services::ProverService;
6+
37
use super::ArtifactBuilder;
48
use anyhow::{anyhow, Context};
59
use mithril_common::{
@@ -11,12 +15,14 @@ use mithril_common::{
1115
};
1216

1317
/// A [CardanoTransactionsArtifact] builder
14-
pub struct CardanoTransactionsArtifactBuilder {}
18+
pub struct CardanoTransactionsArtifactBuilder {
19+
prover_service: Arc<dyn ProverService>,
20+
}
1521

1622
impl CardanoTransactionsArtifactBuilder {
1723
/// CardanoTransactions artifact builder factory
18-
pub fn new() -> Self {
19-
Self {}
24+
pub fn new(prover_service: Arc<dyn ProverService>) -> Self {
25+
Self { prover_service }
2026
}
2127
}
2228

@@ -41,6 +47,7 @@ impl ArtifactBuilder<CardanoDbBeacon, CardanoTransactionsSnapshot>
4147
SignedEntityType::CardanoTransactions(beacon.clone())
4248
)
4349
})?;
50+
self.prover_service.compute_cache(&beacon).await?;
4451

4552
Ok(CardanoTransactionsSnapshot::new(
4653
merkle_root.to_string(),
@@ -51,10 +58,9 @@ impl ArtifactBuilder<CardanoDbBeacon, CardanoTransactionsSnapshot>
5158

5259
#[cfg(test)]
5360
mod tests {
54-
use mithril_common::{
55-
entities::ProtocolMessage,
56-
test_utils::fake_data::{self},
57-
};
61+
use mithril_common::{entities::ProtocolMessage, test_utils::fake_data};
62+
63+
use crate::services::MockProverService;
5864

5965
use super::*;
6066

@@ -72,7 +78,10 @@ mod tests {
7278
};
7379

7480
let beacon = certificate.as_cardano_db_beacon();
75-
let cardano_transaction_artifact_builder = CardanoTransactionsArtifactBuilder::new();
81+
let mut mock_prover = MockProverService::new();
82+
mock_prover.expect_compute_cache().returning(|_| Ok(()));
83+
let cardano_transaction_artifact_builder =
84+
CardanoTransactionsArtifactBuilder::new(Arc::new(mock_prover));
7685
let artifact = cardano_transaction_artifact_builder
7786
.compute_artifact(beacon.clone(), &certificate)
7887
.await
@@ -90,7 +99,10 @@ mod tests {
9099
certificate
91100
};
92101

93-
let cardano_transaction_artifact_builder = CardanoTransactionsArtifactBuilder::new();
102+
let mut mock_prover = MockProverService::new();
103+
mock_prover.expect_compute_cache().returning(|_| Ok(()));
104+
let cardano_transaction_artifact_builder =
105+
CardanoTransactionsArtifactBuilder::new(Arc::new(mock_prover));
94106
cardano_transaction_artifact_builder
95107
.compute_artifact(CardanoDbBeacon::default(), &certificate)
96108
.await

‎mithril-aggregator/src/configuration.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,9 @@ pub struct Configuration {
153153
///
154154
/// Will be ignored on (pre)production networks.
155155
pub allow_unparsable_block: bool,
156+
157+
/// Cardano transactions prover cache pool size
158+
pub cardano_transactions_prover_cache_pool_size: usize,
156159
}
157160

158161
/// Uploader needed to copy the snapshot once computed.
@@ -225,6 +228,7 @@ impl Configuration {
225228
cexplorer_pools_url: None,
226229
signer_importer_run_interval: 1,
227230
allow_unparsable_block: false,
231+
cardano_transactions_prover_cache_pool_size: 3,
228232
}
229233
}
230234

@@ -358,6 +362,9 @@ pub struct DefaultConfiguration {
358362
///
359363
/// Will be ignored on (pre)production networks.
360364
pub allow_unparsable_block: String,
365+
366+
/// Cardano transactions prover cache pool size
367+
pub cardano_transactions_prover_cache_pool_size: u32,
361368
}
362369

363370
impl Default for DefaultConfiguration {
@@ -378,6 +385,7 @@ impl Default for DefaultConfiguration {
378385
snapshot_use_cdn_domain: "false".to_string(),
379386
signer_importer_run_interval: 720,
380387
allow_unparsable_block: "false".to_string(),
388+
cardano_transactions_prover_cache_pool_size: 10,
381389
}
382390
}
383391
}
@@ -452,6 +460,10 @@ impl Source for DefaultConfiguration {
452460
"allow_unparsable_block".to_string(),
453461
into_value(myself.allow_unparsable_block),
454462
);
463+
result.insert(
464+
"cardano_transactions_prover_cache_pool_size".to_string(),
465+
into_value(myself.cardano_transactions_prover_cache_pool_size),
466+
);
455467

456468
Ok(result)
457469
}

0 commit comments

Comments
 (0)