feat: CON-1646 Make ECDSA/Schnorr request queue size dynamic with available pre-signatures (#9059)

We can now store significantly more pre-signatures in advance. The rate
at which these pre-signatures may be consumed is currently determined by
the registry's `max_queue_size` config parameter:

If there are _many_ pre-signatures available, increasing this parameter
allows the replica to accept more signature requests that will be paired
with these pre-signatures. This means pre-signatures will be consumed at
a higher rate, leading to better throughput.

However, if there are _no_ pre-signatures available (i.e. because the
stash was depleted), then increasing the queue size implies a higher
worst case latency for requests sitting at the end of the queue. These
requests have to wait until enough pre-signatures were generated for all
the requests sitting in front.

Therefore, the `max_queue_size` parameter currently represents a
trade-off between maximum throughput and worst case latency.

The solution in this PR is to make the queue size dynamic: As long as
there are still enough pre-signatures available, we accept more
signature requests. If there are no pre-signatures available, then we
only accept at most `max_queue_size` requests, as before.

Since we don't want to accept too many requests, even if there are more
pre-signatures available, the dynamic queue size is additionally capped
by a new constant, that is currently set to 100 requests.

For reference, on mainnet, we currently store 100 pre-signatures per
key. The `max_queue_size` is set to 20 for ECDSA and 30 for Schnorr.

Author: eichhorl

rs/execution_environment/src/execution_environment.rs

@@ -33,7 +33,9 @@ use ic_error_types::{ErrorCode, RejectCode, UserError};
 use ic_interfaces::execution_environment::{
     ExecutionMode, IngressHistoryWriter, RegistryExecutionSettings, SubnetAvailableMemory,
 };
-use ic_limits::{LOG_CANISTER_OPERATION_CYCLES_THRESHOLD, SMALL_APP_SUBNET_MAX_SIZE};
+use ic_limits::{
+    LOG_CANISTER_OPERATION_CYCLES_THRESHOLD, MAX_PAIRED_PRE_SIGNATURES, SMALL_APP_SUBNET_MAX_SIZE,
+};
 use ic_logger::{ReplicaLogger, error, info, warn};
 use ic_management_canister_types_private::{
     CanisterChangeOrigin, CanisterHttpRequestArgs, CanisterIdRecord, CanisterInfoRequest,
@@ -61,16 +63,17 @@ use ic_replicated_state::{
         system_state::{CyclesUseCase, PausedExecutionId},
     },
     metadata_state::subnet_call_context_manager::{
-        EcdsaArguments, InstallCodeCall, InstallCodeCallId, ReshareChainKeyContext,
-        SchnorrArguments, SetupInitialDkgContext, SignWithThresholdContext, StopCanisterCall,
-        SubnetCallContext, ThresholdArguments, VetKdArguments,
+        EcdsaArguments, InstallCodeCall, InstallCodeCallId, PreSignatureStash,
+        ReshareChainKeyContext, SchnorrArguments, SetupInitialDkgContext, SignWithThresholdContext,
+        StopCanisterCall, SubnetCallContext, ThresholdArguments, VetKdArguments,
     },
 };
 use ic_types::{
     CanisterId, Cycles, ExecutionRound, Height, NumBytes, NumInstructions, RegistryVersion,
     ReplicaVersion, SubnetId, Time,
     batch::{CanisterCyclesCostSchedule, ChainKeyData},
     canister_http::{CanisterHttpRequestContext, MAX_CANISTER_HTTP_RESPONSE_BYTES},
+    consensus::idkg::IDkgMasterPublicKeyId,
     crypto::{
         ExtendedDerivationPath,
         canister_threshold_sig::{MasterPublicKey, PublicKey},
@@ -3547,7 +3550,7 @@ impl ExecutionEnvironment {
         mut request: Request,
         args: ThresholdArguments,
         derivation_path: Vec<Vec<u8>>,
-        max_queue_size: u32,
+        max_queue_size_registry: u32,
         state: &mut ReplicatedState,
         rng: &mut dyn RngCore,
         subnet_size: usize,
@@ -3624,12 +3627,18 @@ impl ExecutionEnvironment {
             ));
         }
 
+        let dynamic_queue_size = get_dynamic_signature_queue_size(
+            state.pre_signature_stashes(),
+            max_queue_size_registry,
+            &threshold_key,
+        );
+
         // Check if the queue is full.
         if state
             .metadata
             .subnet_call_context_manager
             .sign_with_threshold_contexts_count(&threshold_key)
-            >= max_queue_size as usize
+            >= dynamic_queue_size
         {
             return Err(UserError::new(
                 ErrorCode::CanisterRejectedMessage,
@@ -4792,3 +4801,29 @@ fn get_master_public_key<'a>(
         Some(master_key) => Ok(master_key),
     }
 }
+
+/// Returns the dynamic signature request queue size for the given key.
+/// In case the stash contains many pre-signatures for the requested key, then we increase
+/// the queue size dynamically up to the maximum number of paired pre-signatures.
+/// This is because a signature is much easier to produce if there is already a pre-signature
+/// available.
+pub(crate) fn get_dynamic_signature_queue_size(
+    stashes: &BTreeMap<IDkgMasterPublicKeyId, PreSignatureStash>,
+    max_queue_size_registry: u32,
+    key_id: &MasterPublicKeyId,
+) -> usize {
+    if let Ok(key_id) = IDkgMasterPublicKeyId::try_from(key_id.clone()) {
+        // If this key uses pre-signatures, we can accept more requests if there are unpaired
+        // pre-signatures available in the stash.
+        let stash_size = stashes
+            .get(&key_id)
+            .map(|stash| stash.pre_signatures.len())
+            .unwrap_or_default();
+        // We never want to allow more requests than the maximum number of paired pre-signatures.
+        let max_queue_size = MAX_PAIRED_PRE_SIGNATURES.min(max_queue_size_registry as usize);
+        stash_size.clamp(max_queue_size, MAX_PAIRED_PRE_SIGNATURES)
+    } else {
+        // If this key doesn't use pre-signatures, we use the registry's max queue size.
+        max_queue_size_registry as usize
+    }
+}

rs/execution_environment/src/execution_environment/tests.rs

@@ -3,6 +3,7 @@ use candid::{Decode, Encode};
 use ic_base_types::{NumBytes, NumSeconds};
 use ic_btc_interface::NetworkInRequest;
 use ic_error_types::{ErrorCode, RejectCode, UserError};
+use ic_limits::MAX_PAIRED_PRE_SIGNATURES;
 use ic_management_canister_types_private::{
     self as ic00, BitcoinGetUtxosArgs, BoundedHttpHeaders, CanisterChange, CanisterHttpRequestArgs,
     CanisterIdRecord, CanisterMetadataRequest, CanisterMetadataResponse, CanisterStatusResultV2,
@@ -19,10 +20,12 @@ use ic_replicated_state::{
     canister_state::{
         DEFAULT_QUEUE_CAPACITY, WASM_PAGE_SIZE_IN_BYTES, system_state::CyclesUseCase,
     },
+    metadata_state::subnet_call_context_manager::PreSignatureStash,
     metadata_state::testing::NetworkTopologyTesting,
     testing::{CanisterQueuesTesting, SystemStateTesting},
 };
 use ic_test_utilities::assert_utils::assert_balance_equals;
+use ic_test_utilities_consensus::idkg::fake_pre_signature_stash;
 use ic_test_utilities_execution_environment::{
     ExecutionTest, ExecutionTestBuilder, check_ingress_status, expect_canister_did_not_reply,
     get_reject, get_reply,
@@ -32,6 +35,7 @@ use ic_types::{
     CanisterId, CountBytes, Cycles, PrincipalId, RegistryVersion,
     batch::CanisterCyclesCostSchedule,
     canister_http::{CanisterHttpMethod, Transform},
+    consensus::idkg::IDkgMasterPublicKeyId,
     ingress::{IngressState, IngressStatus, WasmResult},
     messages::{
         CallbackId, MAX_RESPONSE_COUNT_BYTES, NO_DEADLINE, Payload, RejectContext,
@@ -44,6 +48,7 @@ use ic_types_test_utils::ids::{canister_test_id, node_test_id, subnet_test_id, u
 use ic_universal_canister::{CallArgs, UNIVERSAL_CANISTER_WASM, call_args, wasm};
 use maplit::btreemap;
 use more_asserts::{assert_ge, assert_gt, assert_le, assert_lt};
+use std::collections::BTreeMap;
 use std::mem::size_of;
 
 #[cfg(test)]
@@ -4605,3 +4610,85 @@ fn cannot_accept_cycles_after_replying() {
         initial_cycles + (transferred_cycles / 2u64)
     );
 }
+
+#[test]
+fn get_dynamic_signature_queue_size_vetkd_returns_registry_max() {
+    // Keys that don't use pre-signatures (e.g. VetKd) get the registry max queue size.
+    let key_id = make_vetkd_key("vetkd_key");
+    let stashes: BTreeMap<IDkgMasterPublicKeyId, PreSignatureStash> = BTreeMap::new();
+    assert_eq!(
+        super::get_dynamic_signature_queue_size(&stashes, 50, &key_id),
+        50
+    );
+    assert_eq!(
+        super::get_dynamic_signature_queue_size(&stashes, 200, &key_id),
+        200
+    );
+}
+
+#[test]
+fn get_dynamic_signature_queue_size_idkg_empty_stash_returns_min_of_registry_and_cap() {
+    // Ecdsa/Schnorr key with no stash: queue size is min(registry, MAX_PAIRED_PRE_SIGNATURES).
+    let key_id = make_ecdsa_key("ecdsa_test_key");
+    let stashes: BTreeMap<IDkgMasterPublicKeyId, PreSignatureStash> = BTreeMap::new();
+
+    // Registry < constant → result is registry limit.
+    let max_queue_size_registry = MAX_PAIRED_PRE_SIGNATURES as u32 - 50;
+    assert_eq!(
+        super::get_dynamic_signature_queue_size(&stashes, max_queue_size_registry, &key_id),
+        max_queue_size_registry as usize
+    );
+    // Registry > contant → result is the constant (capped).
+    let max_queue_size_registry = MAX_PAIRED_PRE_SIGNATURES as u32 + 50;
+    assert_eq!(
+        super::get_dynamic_signature_queue_size(&stashes, max_queue_size_registry, &key_id),
+        MAX_PAIRED_PRE_SIGNATURES
+    );
+}
+
+#[test]
+fn get_dynamic_signature_queue_size_idkg_stash_size_within_range() {
+    // Stash size in [max_queue_size, MAX_PAIRED_PRE_SIGNATURES] is returned as-is.
+    let ecdsa_key_id = IDkgMasterPublicKeyId::try_from(make_ecdsa_key("ecdsa_stash_test")).unwrap();
+    let key_id = ecdsa_key_id.inner();
+    let pre_signature_count = 30;
+    let stash = fake_pre_signature_stash(&ecdsa_key_id, pre_signature_count);
+    let stashes = BTreeMap::from([(ecdsa_key_id.clone(), stash)]);
+
+    // Registry 20, stash 30 → max_queue_size=20, result = clamp(30, 20, 100) = 30.
+    assert_eq!(
+        super::get_dynamic_signature_queue_size(&stashes, 20, key_id),
+        pre_signature_count as usize
+    );
+}
+
+#[test]
+fn get_dynamic_signature_queue_size_idkg_stash_size_capped_at_max_paired() {
+    // Stash size > MAX_PAIRED_PRE_SIGNATURES is capped.
+    let ecdsa_key_id =
+        IDkgMasterPublicKeyId::try_from(make_ecdsa_key("ecdsa_large_stash")).unwrap();
+    let key_id = ecdsa_key_id.inner();
+    let stash = fake_pre_signature_stash(&ecdsa_key_id, 150);
+    let stashes = BTreeMap::from([(ecdsa_key_id.clone(), stash)]);
+
+    assert_eq!(
+        super::get_dynamic_signature_queue_size(&stashes, 20, key_id),
+        MAX_PAIRED_PRE_SIGNATURES
+    );
+}
+
+#[test]
+fn get_dynamic_signature_queue_size_idkg_stash_below_floor_returns_floor() {
+    // Stash size < max_queue_size → result is max_queue_size (floor).
+    let ecdsa_key_id =
+        IDkgMasterPublicKeyId::try_from(make_ecdsa_key("ecdsa_small_stash")).unwrap();
+    let key_id = ecdsa_key_id.inner();
+    let stash = fake_pre_signature_stash(&ecdsa_key_id, 5);
+    let stashes = BTreeMap::from([(ecdsa_key_id.clone(), stash)]);
+
+    // Registry 50, stash 5 → max_queue_size=50, result = clamp(5, 50, 100) = 50.
+    assert_eq!(
+        super::get_dynamic_signature_queue_size(&stashes, 50, key_id),
+        50
+    );
+}

rs/limits/src/lib.rs

@@ -96,3 +96,7 @@ pub const DKG_DEALINGS_PER_BLOCK: usize = 1;
 /// Please note that we put fairly big number mainly for perfomance reasons so either side of a channel doesn't await.
 /// The replica code should be designed in such a way that if we put a channel of size 1, the protocol should still work.
 pub const MAX_P2P_IO_CHANNEL_SIZE: usize = 100_000;
+
+/// The maximum number of pre-signatures that may be paired with signature requests,
+/// per key ID.
+pub const MAX_PAIRED_PRE_SIGNATURES: usize = 100;