perf: CON-1657 Validate HTTPS outcall signatures in parallel (#11450)

The main bottleneck of validating blocks with many HTTPS outcalls
responses is signature verification. Therefore, in
#10345 we started using batch
verification which has better performance than validating signatures
one-by-one.

With this PR we additionally pass a rayon thread pool to the HTTPS
outcalls payload builder. This thread pool has 16 threads, and is
already used by other payload builders to reduce validation times (the
`ChainKeyPayloadBuilder` for instance). Similar to that, we use the
thread pool to split all signatures of the current block in to at most
16 batches (one for each thread), and then validate each batch in
parallel, with a single batch verification call. This reduces the
validation time of full HTTPS outcalls payloads by up to ~80%.

### Before
```sql
canister_http_payload_verification/mixed_subnet34
                        time:   [107.00 ms 107.66 ms 108.38 ms]
canister_http_payload_verification/many_replicated_responses_subnet34
                        time:   [149.01 ms 149.87 ms 150.85 ms]
canister_http_payload_verification/many_non_replicated_responses_subnet34
                        time:   [8.2811 ms 8.3068 ms 8.3353 ms]
canister_http_payload_verification/many_divergence_responses_subnet34
                        time:   [79.840 ms 80.548 ms 81.381 ms]
canister_http_payload_verification/many_flexible_responses_subnet34
                        time:   [179.01 ms 180.38 ms 181.82 ms]
```
### After
```sql
canister_http_payload_verification/mixed_subnet34
                        time:   [21.519 ms 21.788 ms 22.066 ms]
                        change: [-80.032% -79.762% -79.482%] (p = 0.00 < 0.05)
                        Performance has improved.
canister_http_payload_verification/many_replicated_responses_subnet34
                        time:   [22.973 ms 23.205 ms 23.457 ms]
                        change: [-84.695% -84.517% -84.330%] (p = 0.00 < 0.05)
                        Performance has improved.
canister_http_payload_verification/many_non_replicated_responses_subnet34
                        time:   [3.0804 ms 3.1072 ms 3.1346 ms]
                        change: [-62.932% -62.595% -62.245%] (p = 0.00 < 0.05)
                        Performance has improved.
canister_http_payload_verification/many_divergence_responses_subnet34
                        time:   [14.198 ms 14.318 ms 14.452 ms]
                        change: [-82.456% -82.225% -81.996%] (p = 0.00 < 0.05)
                        Performance has improved.
canister_http_payload_verification/many_flexible_responses_subnet34
                        time:   [43.591 ms 43.947 ms 44.336 ms]
                        change: [-75.914% -75.637% -75.337%] (p = 0.00 < 0.05)
                        Performance has improved.
```

---------

Co-authored-by: IDX GitHub Automation <infra+github-automation@dfinity.org>
Co-authored-by: Pierugo Pace <pierugo.pace@dfinity.org>

Author: 3 people

Cargo.lock

@@ -658,6 +658,7 @@ mod tests {
     use assert_matches::assert_matches;
     use core::{convert::From, iter::Iterator, time::Duration};
     use ic_consensus_mocks::{Dependencies, DependenciesBuilder};
+    use ic_consensus_utils::build_thread_pool;
     use ic_crypto_temp_crypto::TempCryptoComponent;
     use ic_interfaces::consensus::{InvalidPayloadReason, PayloadValidationFailure};
     use ic_interfaces::idkg::IDkgChangeAction;
@@ -676,7 +677,6 @@ mod tests {
     use ic_types::time::UNIX_EPOCH;
     use ic_types::time::current_time;
     use ic_types_test_utils::ids::{node_test_id, subnet_test_id};
-    use rayon::ThreadPoolBuilder;
     use std::str::FromStr;
 
     use super::*;
@@ -873,12 +873,7 @@ mod tests {
                 pool.get_cache(),
                 crypto,
                 state_manager,
-                Arc::new(
-                    ThreadPoolBuilder::new()
-                        .num_threads(num_threads)
-                        .build()
-                        .unwrap(),
-                ),
+                build_thread_pool(num_threads),
                 subnet_id,
                 registry,
                 &MetricsRegistry::new(),

rs/consensus/chain_key/src/lib.rs

@@ -185,9 +185,11 @@ use crate::{
     metrics::{CRITICAL_ERROR_IDKG_RETAIN_ACTIVE_TRANSCRIPTS, IDkgClientMetrics, timed_call},
     pre_signer::{IDkgPreSigner, IDkgPreSignerImpl},
     signer::{ThresholdSigner, ThresholdSignerImpl},
-    utils::{IDkgBlockReaderImpl, build_thread_pool},
+    utils::IDkgBlockReaderImpl,
+};
+use ic_consensus_utils::{
+    bouncer_metrics::BouncerMetrics, build_thread_pool, crypto::ConsensusCrypto,
 };
-use ic_consensus_utils::{bouncer_metrics::BouncerMetrics, crypto::ConsensusCrypto};
 use ic_interfaces::{
     consensus_pool::{ConsensusBlockCache, ConsensusPoolCache},
     crypto::IDkgProtocol,

rs/consensus/idkg/src/lib.rs

@@ -548,9 +548,9 @@ mod test {
             resharing::{initiate_reshare_requests, update_completed_reshare_requests},
         },
         test_utils::*,
-        utils::build_thread_pool,
     };
     use assert_matches::assert_matches;
+    use ic_consensus_utils::build_thread_pool;
     use ic_crypto_temp_crypto::TempCryptoComponent;
     use ic_crypto_test_utils_canister_threshold_sigs::{
         CanisterThresholdSigTestEnvironment, dummy_values::dummy_dealings,

rs/consensus/idkg/src/payload_verifier.rs

@@ -3,12 +3,11 @@ use crate::{
     complaints::{IDkgComplaintHandlerImpl, IDkgTranscriptLoader, TranscriptLoadStatus},
     pre_signer::{IDkgPreSignerImpl, IDkgTranscriptBuilder},
     signer::ThresholdSignerImpl,
-    utils::build_thread_pool,
 };
 use ic_artifact_pool::idkg_pool::IDkgPoolImpl;
 use ic_config::artifact_pool::ArtifactPoolConfig;
 use ic_consensus_mocks::{Dependencies, DependenciesBuilder};
-use ic_consensus_utils::crypto::ConsensusCrypto;
+use ic_consensus_utils::{build_thread_pool, crypto::ConsensusCrypto};
 use ic_crypto_temp_crypto::TempCryptoComponent;
 use ic_crypto_test_utils_canister_threshold_sigs::{
     CanisterThresholdSigTestEnvironment, IDkgParticipants, IntoBuilder,

rs/consensus/idkg/src/test_utils.rs

@@ -32,7 +32,6 @@ use ic_types::{
     },
     registry::RegistryClientError,
 };
-use rayon::{ThreadPool, ThreadPoolBuilder};
 use std::{
     cell::RefCell,
     collections::{BTreeMap, BTreeSet},
@@ -500,16 +499,6 @@ impl<T: Ord + Copy> IDkgSchedule<T> {
     }
 }
 
-/// Builds a rayon thread pool with the given number of threads.
-pub(crate) fn build_thread_pool(num_threads: usize) -> Arc<ThreadPool> {
-    Arc::new(
-        ThreadPoolBuilder::new()
-            .num_threads(num_threads)
-            .build()
-            .expect("Failed to create thread pool"),
-    )
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

rs/consensus/idkg/src/utils.rs

@@ -59,7 +59,7 @@ use ic_types::{
     Time, artifact::ConsensusMessageId, consensus::ConsensusMessageHashable,
     malicious_flags::MaliciousFlags, replica_config::ReplicaConfig,
 };
-use rayon::{ThreadPool, ThreadPoolBuilder};
+use rayon::ThreadPool;
 use std::{
     cell::RefCell,
     collections::BTreeMap,
@@ -82,9 +82,6 @@ pub const ACCEPTABLE_NOTARIZATION_CERTIFICATION_GAP: u64 = 70;
 /// CUPs, which have no upper bound on the height to be validated.
 pub(crate) const ACCEPTABLE_NOTARIZATION_CUP_GAP: u64 = 130;
 
-/// The maximum number of threads used to create & validate block payloads in parallel.
-pub const MAX_CONSENSUS_THREADS: usize = 16;
-
 #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug, AsRefStr)]
 #[strum(serialize_all = "snake_case")]
 enum ConsensusSubcomponent {
@@ -99,16 +96,6 @@ enum ConsensusSubcomponent {
     Purger,
 }
 
-/// Builds a rayon thread pool with the given number of threads.
-pub fn build_thread_pool(num_threads: usize) -> Arc<ThreadPool> {
-    Arc::new(
-        ThreadPoolBuilder::new()
-            .num_threads(num_threads)
-            .build()
-            .expect("Failed to create thread pool"),
-    )
-}
-
 /// [ConsensusImpl] holds all consensus subcomponents, and implements the
 /// Consensus trait by calling each subcomponent in round-robin manner.
 pub struct ConsensusImpl {
@@ -622,6 +609,7 @@ mod tests {
     use super::*;
     use ic_config::artifact_pool::ArtifactPoolConfig;
     use ic_consensus_mocks::{Dependencies, DependenciesBuilder};
+    use ic_consensus_utils::{MAX_CONSENSUS_THREADS, build_thread_pool};
     use ic_https_outcalls_consensus::test_utils::FakeCanisterHttpPayloadBuilder;
     use ic_logger::replica_logger::no_op_logger;
     use ic_metrics::MetricsRegistry;

rs/consensus/src/consensus.rs

@@ -2160,12 +2160,12 @@ impl Validator {
 pub mod test {
     use super::*;
     use crate::consensus::{
-        MAX_CONSENSUS_THREADS, block_maker::get_block_maker_delay, build_thread_pool,
-        catchup_package_maker::CatchUpPackageMaker,
+        block_maker::get_block_maker_delay, catchup_package_maker::CatchUpPackageMaker,
     };
     use assert_matches::assert_matches;
     use ic_config::artifact_pool::ArtifactPoolConfig;
     use ic_consensus_mocks::{Dependencies, DependenciesBuilder, RefMockPayloadBuilder};
+    use ic_consensus_utils::{MAX_CONSENSUS_THREADS, build_thread_pool};
     use ic_crypto_test_utils_crypto_returning_ok::CryptoReturningOk;
     use ic_interfaces::{
         messaging::XNetPayloadValidationFailure, p2p::consensus::MutablePool,

rs/consensus/src/consensus/validator.rs

@@ -4,10 +4,9 @@ use ic_artifact_pool::{
     consensus_pool::ConsensusPoolImpl, dkg_pool, idkg_pool,
 };
 use ic_config::artifact_pool::ArtifactPoolConfig;
-use ic_consensus::consensus::{
-    ConsensusBouncer, ConsensusImpl, MAX_CONSENSUS_THREADS, build_thread_pool,
-};
+use ic_consensus::consensus::{ConsensusBouncer, ConsensusImpl};
 use ic_consensus_idkg::IDkgImpl;
+use ic_consensus_utils::{MAX_CONSENSUS_THREADS, build_thread_pool};
 use ic_https_outcalls_consensus::test_utils::FakeCanisterHttpPayloadBuilder;
 use ic_interfaces::{
     batch_payload::BatchPayloadBuilder,

rs/consensus/tests/framework/types.rs

@@ -4,10 +4,9 @@ mod framework;
 use crate::framework::ConsensusDriver;
 use assert_matches::assert_matches;
 use ic_artifact_pool::{consensus_pool, dkg_pool, idkg_pool};
-use ic_consensus::consensus::{MAX_CONSENSUS_THREADS, build_thread_pool};
 use ic_consensus_certification::CertifierImpl;
 use ic_consensus_dkg::{DkgKeyManager, get_dkg_summary_from_cup_contents};
-use ic_consensus_utils::pool_reader::PoolReader;
+use ic_consensus_utils::{MAX_CONSENSUS_THREADS, build_thread_pool, pool_reader::PoolReader};
 use ic_crypto_test_utils_crypto_returning_ok::CryptoReturningOk;
 use ic_https_outcalls_consensus::test_utils::FakeCanisterHttpPayloadBuilder;
 use ic_interfaces_registry::RegistryClient;