chore: refine testing infrastructure and tests exec time comparison

.config/nextest.toml

@@ -31,8 +31,26 @@ path = "junit.xml"
 store-success-output = true
 store-failure-output = true
 
-# Serialize Docker Compose tests: "binary(=integration_test)" is an exact match,
-# so it does NOT affect integration_test_isolated (which runs in parallel).
+# Long timeout for tests that use production-sized FHE parameters (Default +
+# SnS). Give them a 2h window before termination.
+#
+# Docker-based Default-param tests (integration_test binary):
+[[profile.ci.overrides]]
+filter = 'binary(=integration_test) & test(/full_gen_tests_default|nightly_tests_threshold_sequential_crs|test_threshold_concurrent_crs|test_threshold_insecure/)'
+slow-timeout = { period = "600s", terminate-after = 12 }
+test-group = "docker-integration"
+
+# Native isolated Default-param tests (integration_test_isolated binary):
+# These don't use Docker but are placed in docker-integration to prevent
+# CPU contention with Docker Default-param tests running concurrently.
+[[profile.ci.overrides]]
+filter = 'binary(=integration_test_isolated) & test(/full_gen_tests_default|nightly_tests_threshold_sequential_crs|test_threshold_concurrent_crs/)'
+slow-timeout = { period = "600s", terminate-after = 12 }
+test-group = "docker-integration"
+
+# Serialize remaining Docker Compose tests (non-Default-param).
+# "binary(=integration_test)" is an exact match — does NOT affect
+# integration_test_isolated.
 [[profile.ci.overrides]]
 filter = "binary(=integration_test)"
 test-group = "docker-integration"
@@ -43,15 +61,6 @@ test-group = "docker-integration"
 filter = "binary(/^kubernetes_test_/)"
 test-group = "k8s-integration"
 
-# Long timeout for tests that use production-sized FHE parameters (Default +
-# SnS).  These are inherently very slow (keygen/CRS can take 30-90 min).
-# Give them a 2h window before termination and serialize them with the Docker
-# tests to avoid CPU/memory contention on CI machines.
-[[profile.ci.overrides]]
-filter = 'test(/full_gen_tests_default|nightly_tests_threshold_sequential_crs|test_threshold_concurrent_crs/)'
-slow-timeout = { period = "600s", terminate-after = 12 }
-test-group = "docker-integration"
-
 # =============================================================================
 # CI-nightly profile: used for scheduled builds — runs ALL tests
 # =============================================================================
@@ -65,7 +74,22 @@ path = "junit.xml"
 store-success-output = true
 store-failure-output = true
 
-# Serialize Docker Compose tests (same as ci profile).
+# Long timeout + serialization for slow Default-param tests (same as ci profile).
+# IMPORTANT: Must come before docker-integration catch-all (first match wins).
+#
+# Docker-based Default-param tests (integration_test binary):
+[[profile.ci-nightly.overrides]]
+filter = 'binary(=integration_test) & test(/full_gen_tests_default|nightly_tests_threshold_sequential_crs|test_threshold_concurrent_crs|test_threshold_insecure/)'
+slow-timeout = { period = "600s", terminate-after = 12 }
+test-group = "docker-integration"
+
+# Native isolated Default-param tests (integration_test_isolated binary):
+[[profile.ci-nightly.overrides]]
+filter = 'binary(=integration_test_isolated) & test(/full_gen_tests_default|nightly_tests_threshold_sequential_crs|test_threshold_concurrent_crs/)'
+slow-timeout = { period = "600s", terminate-after = 12 }
+test-group = "docker-integration"
+
+# Serialize remaining Docker Compose tests (same as ci profile).
 [[profile.ci-nightly.overrides]]
 filter = "binary(=integration_test)"
 test-group = "docker-integration"
@@ -74,9 +98,3 @@ test-group = "docker-integration"
 [[profile.ci-nightly.overrides]]
 filter = "binary(/^kubernetes_test_/)"
 test-group = "k8s-integration"
-
-# Long timeout + serialization for slow Default-param tests (same as ci profile).
-[[profile.ci-nightly.overrides]]
-filter = 'test(/full_gen_tests_default|nightly_tests_threshold_sequential_crs|test_threshold_concurrent_crs/)'
-slow-timeout = { period = "600s", terminate-after = 12 }
-test-group = "docker-integration"

.github/workflows/common-testing.yml

@@ -76,6 +76,10 @@ on:
         type: boolean
         required: false
         default: false  # If true, download test material from artifacts instead of generating
+      generate-default-material:
+        type: boolean
+        required: false
+        default: false  # If true, generate Default test material (includes pre-generated PRSS)
     secrets:
       CODECOV_TOKEN:
         required: false
@@ -302,12 +306,13 @@ jobs:
           echo "Generating testing material..."
           cargo run -p generate-test-material -- --output ./test-material --verbose testing
 
-      # Generate default material only if slow_tests and not using pre-generated
+      # Generate default material when explicitly requested or for slow_tests,
+      # and only when not using pre-generated artifacts.
       - name: Generate Test Material (Default)
-        if: ${{ ! inputs.use-pregenerated-material && contains(inputs.args-tests, 'slow_tests') }}
+        if: ${{ ! inputs.use-pregenerated-material && (inputs.generate-default-material || contains(inputs.args-tests, 'slow_tests')) }}
         working-directory: .
         run: |
-          echo "Generating default test material for slow_tests..."
+          echo "Generating default test material..."
           cargo run -p generate-test-material --features slow_tests -- --output ./test-material --verbose default
 
       # Build kms-custodian binary required by integration tests

.github/workflows/kind-testing.yml

@@ -366,7 +366,7 @@ jobs:
           #============================================================
           #
           set +e
-          cargo nextest run --test 'kubernetes_test_'"${DEPLOYMENT_TYPE}"'*' --features k8s_tests,testing --profile "${NEXTEST_PROFILE}" --no-fail-fast
+          cargo nextest run --test 'kubernetes_test_'"${DEPLOYMENT_TYPE}"'*' --features kind_tests --profile "${NEXTEST_PROFILE}" --no-fail-fast
           echo "TEST_EXIT_CODE=$?" >> "${GITHUB_ENV}"
           set -e
 

.github/workflows/main.yml

@@ -72,7 +72,7 @@ jobs:
               - 'Cargo.toml'
               - 'Cargo.lock'
             core-client-k8s-tests:
-              - 'core-client/tests/kubernetes_test_*'
+              - 'core-client/tests/**'
             core-grpc:
               - 'core/grpc/**'
               - 'Cargo.toml'
@@ -263,7 +263,7 @@ jobs:
   ############################################################################
   # Prepares test matrix for core-client integration tests
   # Splits tests into two parallel jobs: threshold and centralized
-  # Features required: k8s_tests (for PRSS tests), testing (for test helpers)
+  # Features required: threshold_tests (for PRSS tests), testing (for test helpers)
   prepare-core-client-matrix:
     name: main/prepare-core-client-matrix
     needs: check-changes
@@ -285,7 +285,7 @@ jobs:
                     # Threshold tests: includes PRSS tests (sequential execution via #[serial])
           # Centralized tests: basic keygen, CRS, backup/restore workflows
           # Both skip: full_gen_tests (nightly), nightly_* (scheduled), k8s_* (cluster tests), isolated_test_example (demo)
-          MATRIX="${MATRIX}{\"args-tests\":\"--features testing,k8s_tests -- threshold --skip full_gen_tests --skip nightly --skip k8s_ --skip centralized --skip isolated_test_example\"},{\"args-tests\":\"--features testing -- centralized --skip full_gen_tests --skip nightly --skip k8s_ --skip threshold --skip isolated_test_example\"}"
+          MATRIX="${MATRIX}{\"args-tests\":\"--features threshold_tests -- threshold --skip full_gen_tests --skip nightly --skip k8s_ --skip centralized --skip isolated_test_example\",\"generate_default_material\":true},{\"args-tests\":\"--features testing -- centralized --skip full_gen_tests --skip nightly --skip k8s_ --skip threshold --skip isolated_test_example\",\"generate_default_material\":false}"
           MATRIX="${MATRIX%,}]}"
           echo "$MATRIX" >> "$GITHUB_OUTPUT"
           echo "$MATRIX"
@@ -320,6 +320,7 @@ jobs:
       working-directory: './core-client'
       package-name: 'kms-core-client'
       args-tests: ${{ matrix.args-tests }}
+      generate-default-material: ${{ matrix.generate_default_material }}
       app-cache-dir: 'kms-core-client'
       # Availables runners:
       #  - 1cpu-linux-x64 (m7a.medium)
@@ -363,7 +364,9 @@ jobs:
       working-directory: './core-client'
       package-name: 'kms-core-client'
       # Run ALL tests - no skips (includes nightly_* and full_gen_tests_*)
-      args-tests: '--features testing,k8s_tests -- --skip k8s_ --skip isolated_test_example'
+      # nightly_* and full_gen_tests_* are skipped in regular CI via --skip prefix matching
+      args-tests: '--features threshold_tests -- --skip k8s_ --skip isolated_test_example'
+      generate-default-material: true
       nextest-profile: 'ci-nightly'
       app-cache-dir: 'kms-core-client'
       runs-on: '32cpu-linux-x64'

core-client/Cargo.toml

@@ -16,6 +16,36 @@ path = "src/lib.rs"
 name = "kms-core-client"
 path = "src/bin.rs"
 
+[[test]]
+name = "integration_test"
+path = "tests/integration/integration_test.rs"
+required-features = ["testing"]
+
+[[test]]
+name = "integration_test_isolated"
+path = "tests/integration/integration_test_isolated.rs"
+required-features = ["testing"]
+
+[[test]]
+name = "kubernetes_test_centralized"
+path = "tests/kind-testing/kubernetes_test_centralized.rs"
+required-features = ["kind_tests"]
+
+[[test]]
+name = "kubernetes_test_centralized_isolated"
+path = "tests/kind-testing/kubernetes_test_centralized_isolated.rs"
+required-features = ["kind_tests"]
+
+[[test]]
+name = "kubernetes_test_threshold"
+path = "tests/kind-testing/kubernetes_test_threshold.rs"
+required-features = ["kind_tests"]
+
+[[test]]
+name = "kubernetes_test_threshold_isolated"
+path = "tests/kind-testing/kubernetes_test_threshold_isolated.rs"
+required-features = ["kind_tests"]
+
 [dependencies]
 # ⚠️ IMPORTANT: Add new dependencies to workspace root Cargo.toml, NOT here!
 # Use .workspace = true to reference workspace dependencies
@@ -71,7 +101,21 @@ tonic-build.workspace = true
 ignored = ["kms", "strum"]
 
 [features]
-# Enable PRSS-dependent tests that require sequential execution and stable networking
-# These tests are designed for K8s CI environments
-k8s_tests = []
+# Enables threshold PRSS tests in `integration_test_isolated`.
+# These are the threshold tests that require PRSS infrastructure:
+#   preproc+keygen, MPC context init/switch, reshare, full-gen preproc.
+# Un-ignores tests guarded with `#[cfg_attr(not(feature = "threshold_tests"), ignore)]`.
+# Compiles setup helpers that run threshold servers with `run_prss=true`.
+#
+# This flag only gates code/tests; it does NOT generate key material by itself.
+# Material requirements:
+# - Test params: missing PRSS can be initialized live at server startup.
+# - Default params: pre-generated PRSS is required in `test-material/default`
+#   (run `make generate-test-material-default`); missing PRSS is a hard error.
+# - Some tests generate PRSS during the test itself (via NewEpoch/new_prss_isolated)
+#   and therefore do not require pre-generated startup PRSS.
+# Does NOT enable Kind/Kubernetes tests (use `kind_tests` for those).
+threshold_tests = ["testing"]
+# Enable tests that require a running Kind/Kubernetes cluster.
+kind_tests = ["testing"]
 testing = []

core-client/src/keygen.rs

@@ -591,6 +591,10 @@ pub(crate) async fn get_preproc_keygen_responses(
             ))
             .await;
 
+            tracing::info!(
+                "Polling preproc result for request {} from party {}",
+                request_id, core_conf.party_id
+            );
             let mut response = client
                 .get_key_gen_preproc_result(tonic::Request::new(request_id.into()))
                 .await;
@@ -610,6 +614,10 @@ pub(crate) async fn get_preproc_keygen_responses(
                     );
                 }
                 ctr += 1;
+                tracing::info!(
+                    "Preproc result not ready yet for request {} from party {} (retry {}/{})",
+                    request_id, core_conf.party_id, ctr, max_iter
+                );
                 response = client
                     .get_key_gen_preproc_result(tonic::Request::new(request_id.into()))
                     .await;

core-client/tests-utils/src/lib.rs

@@ -1,8 +1,10 @@
 use std::env;
 use std::fmt;
 use std::io::Write;
+use std::net::TcpStream;
 use std::path::PathBuf;
 use std::process::{Command, Output};
+use std::time::{Duration, Instant};
 
 pub struct DockerComposeCmd {
     pub root_path: PathBuf,
@@ -40,6 +42,21 @@ pub fn format_output(output: &Output) -> OutputWrapper<'_> {
     OutputWrapper(output)
 }
 
+/// Wait until all given TCP ports on localhost are no longer bound.
+/// This prevents "address already in use" errors when Docker Compose retries
+/// start before the OS has released ports from the previous run.
+fn wait_for_ports_free(ports: &[u16], timeout: Duration) {
+    let deadline = Instant::now() + timeout;
+    for &port in ports {
+        while Instant::now() < deadline {
+            if TcpStream::connect(("127.0.0.1", port)).is_err() {
+                break; // port is free
+            }
+            std::thread::sleep(Duration::from_millis(500));
+        }
+    }
+}
+
 impl DockerComposeCmd {
     pub fn new(mode: KMSMode) -> Self {
         let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set");
@@ -51,8 +68,26 @@ impl DockerComposeCmd {
         DockerComposeCmd { root_path, mode }
     }
 
+    fn ports_for_mode(&self) -> &'static [u16] {
+        match self.mode {
+            KMSMode::ThresholdTestParameterNoInitSixParty => &[
+                50100, 50200, 50300, 50400, 50500, 50600, 50001, 50002, 50003, 50004, 50005, 50006,
+            ],
+            KMSMode::ThresholdDefaultParameter
+            | KMSMode::ThresholdTestParameter
+            | KMSMode::ThresholdTestParameterNoInit
+            | KMSMode::ThresholdCustodianTestParameter => {
+                &[50100, 50200, 50300, 50400, 50001, 50002, 50003, 50004]
+            }
+            KMSMode::Centralized | KMSMode::CentralizedCustodian => &[50100],
+        }
+    }
+
     pub fn up(&self) {
         self.down(); // Make sure that no container is running
+                     // Wait for the OS to release ports before starting new containers.
+                     // Without this, Docker Compose retries fail with "address already in use".
+        wait_for_ports_free(self.ports_for_mode(), Duration::from_secs(30));
         let build_docker = env::var("DOCKER_BUILD_TEST_CORE_CLIENT").unwrap_or("".to_string());
 
         // set the FHE params based on mode