test: rejoin test with slow catch-up

rs/execution_environment/Contributing.md

@@ -41,7 +41,6 @@ The public API of the Management Canister is defined in Candid. Candid is design
 8. Write tests to cover the new or updated functionality:
    - Use the `ExecutionTest` framework by default.
    - Use the `StateMachine` framework if the feature involves inter-canister calls, canister HTTPS outcalls, threshold signatures, or checkpointing. These require mocked Consensus layer outputs or a full state manager.
-   - Make sure to add new scenarios to `rs/execution_environment/tests/memory_matrix.rs` if your change introduces new code paths dealing with canister memory usage/allocation.
 
 9. Once the *Interface Specification* change has been agreed on, the public Management Canister [types](https://crates.io/crates/ic-management-canister-types), [Motoko](https://github.com/dfinity/motoko), and [Rust CDK](https://github.com/dfinity/cdk-rs) can be updated to use the new API on a feature branch. Coordinate with *@eng-sdk* and *@eng-motoko* as needed. The new functionality is enabled for testing in PocketIC (on a PocketIC instance created with `enable_beta_features` set) by enabling the corresponding feature flags in `rs/pocket_ic_server/src/beta_features.rs`.
 

rs/execution_environment/tests/memory_matrix.rs

@@ -7,28 +7,17 @@ It defines multiple *scenarios* and their expectations in terms of memory usage
 and performs multiple *runs* of every scenarios with various initial parameters.
 
 The runs ensure the following properties for every scenario:
-- reserved cycles and subnet available memory are updated properly in both successful and failed executions;
+- subnet available memory is updated properly in both successful and failed executions;
 - the execution fails if the subnet memory capacity would be exceeded;
 - the execution fails if the reserved cycles limit would be exceeded;
 - the execution fails if the canister would become frozen;
 - the execution fails if the canister does not have sufficient balance to reserve storage cycles;
 - the execution does not allocate additional memory for canisters with memory allocation.
 
-Every memory matrix test has the following components:
-- a "setup" function takes `&mut ExecutionTest` and `CanisterId` of an empty canister in that `ExecutionTest`,
-  performs a setup of that canister, and returns arbitrary data of choice (could also be `()` if no data are needed)
-  that are relayed to the "operation" function;
-- an "operation" function takes `&mut ExecutionTest`, `CanisterId` of the canister set up before, and
-  the data produced by the "setup" function before;
-- an instance of `ScenarioParams` also containing `Scenario` and `MemoryUsageChange` describing
-  the kind of scenario and its expectations in terms of canister memory usage change;
-- an actual invokation of the matrix test suite implemented by the function `test_memory_suite`.
-
-The existing scenarios cover the following:
+The scenarios cover the following:
 - growing WASM/stable memory in canister (update) entry point;
 - growing WASM/stable memory in canister reply/cleanup callback;
 - taking a canister snapshot (both growing and shrinking canister memory usage);
-- taking a canister snapshot and uninstalling code atomically;
 - replacing a canister snapshot by a snapshot of the same size;
 - loading a canister snapshot (both growing and shrinking canister memory usage);
 - deleting a canister snapshot;

rs/rust_canisters/statesync_test/BUILD.bazel

@@ -7,6 +7,7 @@ package(default_visibility = ["//visibility:public"])
 DEPENDENCIES = [
     # Keep sorted.
     "@crate_index//:candid",
+    "@crate_index//:futures",
     "@crate_index//:ic-cdk",
     "@crate_index//:lazy_static",
     "@crate_index//:rand",

rs/rust_canisters/statesync_test/Cargo.toml

@@ -12,6 +12,7 @@ path = "src/main.rs"
 
 [dependencies]
 candid = { workspace = true }
+futures = { workspace = true }
 ic-cdk = { workspace = true }
 lazy_static = { workspace = true }
 rand = { workspace = true }

rs/rust_canisters/statesync_test/src/main.rs

@@ -1,3 +1,7 @@
+use futures::{StreamExt, stream};
+use ic_cdk::management_canister::{
+    ProvisionalCreateCanisterWithCyclesArgs, provisional_create_canister_with_cycles,
+};
 use ic_cdk::stable::{
     WASM_PAGE_SIZE_IN_BYTES as PAGE_SIZE, stable_grow, stable_size, stable_write,
 };
@@ -78,4 +82,28 @@ async fn read_state(index: usize) -> Result<u8, String> {
     }
 }
 
+#[update]
+async fn create_many_canisters(n: u64) {
+    let mut futs = vec![];
+
+    for _ in 0..n {
+        let fut = async {
+            let create_args = ProvisionalCreateCanisterWithCyclesArgs {
+                amount: None,
+                settings: None,
+                specified_id: None,
+            };
+            provisional_create_canister_with_cycles(&create_args)
+                .await
+                .expect("Failed to create canister");
+        };
+        futs.push(fut);
+    }
+
+    stream::iter(futs)
+        .buffer_unordered(500) // limit concurrency to 500 (inter-canister queue capacity)
+        .collect::<Vec<_>>()
+        .await;
+}
+
 fn main() {}

rs/rust_canisters/statesync_test/test/test.rs

@@ -3,24 +3,26 @@ use canister_test::*;
 use ic_management_canister_types_private::CanisterSettingsArgsBuilder;
 use ic_state_machine_tests::StateMachine;
 
+fn deploy_state_sync_test_canister(env: &StateMachine) -> CanisterId {
+    let features = [];
+    let wasm = Project::cargo_bin_maybe_from_env("statesync-test-canister", &features);
+    env.install_canister(
+        wasm.bytes(),
+        vec![],
+        Some(
+            CanisterSettingsArgsBuilder::new()
+                .with_memory_allocation(8 * 1024 * 1024 * 1024)
+                .build(),
+        ),
+    )
+    .expect("Failed to install canister")
+}
+
 #[test]
 fn test_statesync_test_canisters() {
     let env = StateMachine::new();
 
-    let features = [];
-    let wasm = Project::cargo_bin_maybe_from_env("statesync-test-canister", &features);
-    let canister_id = env
-        .install_canister(
-            wasm.bytes(),
-            vec![],
-            Some(
-                CanisterSettingsArgsBuilder::new()
-                    .with_memory_allocation(8 * 1024 * 1024 * 1024)
-                    .build(),
-            ),
-        )
-        .expect("Failed to install canister");
-
+    let canister_id = deploy_state_sync_test_canister(&env);
     let result = env
         .query(canister_id, "read_state", Encode!(&0usize).unwrap())
         .unwrap();
@@ -56,6 +58,26 @@ fn test_statesync_test_canisters() {
     );
 }
 
+#[test]
+fn test_create_many_canisters() {
+    let env = StateMachine::new();
+
+    let seed_canister_id = deploy_state_sync_test_canister(&env);
+
+    let num_canisters: u64 = 1000;
+    let result = env
+        .execute_ingress(
+            seed_canister_id,
+            "create_many_canisters",
+            Encode!(&num_canisters).unwrap(),
+        )
+        .unwrap();
+    let _ = assert_reply(result);
+
+    // We created `num_canisters` in addition to the seed canister.
+    assert_eq!(env.num_running_canisters(), num_canisters + 1);
+}
+
 fn assert_reply(res: WasmResult) -> Vec<u8> {
     match res {
         WasmResult::Reply(res) => res,

rs/tests/driver/src/util.rs

@@ -947,6 +947,8 @@ pub async fn agent_with_client_identity(
         .with_url(url)
         .with_http_client(client)
         .with_identity(identity)
+        // Setting a large polling time for the sake of long-running update calls.
+        .with_max_polling_time(Duration::from_secs(3600))
         .with_max_concurrent_requests(MAX_CONCURRENT_REQUESTS)
         // Ingresses are created with the system time but are checked against the consensus time.
         // Consensus time is the time that is in the last finalized block. Consensus time might lag

rs/tests/message_routing/BUILD.bazel

@@ -94,6 +94,29 @@ system_test_nns(
     ],
 )
 
+system_test_nns(
+    name = "rejoin_test_long_rounds",
+    enable_head_nns_variant = False,  # only run this test with the mainnet NNS canisters.
+    env = UNIVERSAL_CANISTER_ENV | {
+        "STATESYNC_TEST_CANISTER_WASM_PATH": "$(rootpath //rs/rust_canisters/statesync_test:statesync-test-canister)",
+    },
+    tags = [
+        "manual",  # this test does not pass currently (it serves as a baseline for future work)
+        "system_test_large",
+    ],
+    test_timeout = "eternal",
+    runtime_deps =
+        STATESYNC_TEST_CANISTER_RUNTIME_DEPS +
+        UNIVERSAL_CANISTER_RUNTIME_DEPS,
+    deps = [
+        "//rs/registry/subnet_type",
+        "//rs/tests/driver:ic-system-test-driver",
+        "//rs/tests/message_routing/rejoin_test_lib",
+        "//rs/types/types",
+        "@crate_index//:anyhow",
+    ],
+)
+
 system_test_nns(
     name = "state_sync_malicious_chunk_test",
     enable_head_nns_variant = False,  # only run this test with the mainnet NNS canisters.

rs/tests/message_routing/Cargo.toml

@@ -42,6 +42,10 @@ path = "rejoin_test.rs"
 name = "rejoin_test_large_state"
 path = "rejoin_test_large_state.rs"
 
+[[bin]]
+name = "rejoin_test_long_rounds"
+path = "rejoin_test_long_rounds.rs"
+
 [[bin]]
 name = "state_sync_malicious_chunk_test"
 path = "state_sync_malicious_chunk_test.rs"

rs/tests/message_routing/rejoin_test_lib/BUILD.bazel

@@ -11,10 +11,14 @@ rust_library(
         "//rs/rust_canisters/canister_test",
         "//rs/rust_canisters/dfn_candid",
         "//rs/tests/driver:ic-system-test-driver",
+        "//rs/types/types",
+        "//rs/universal_canister/lib",
         "@crate_index//:anyhow",
         "@crate_index//:candid",
         "@crate_index//:chrono",
         "@crate_index//:futures",
+        "@crate_index//:ic-agent",
+        "@crate_index//:ic-utils",
         "@crate_index//:slog",
         "@crate_index//:tokio",
     ],

rs/tests/message_routing/rejoin_test_lib/Cargo.toml

@@ -12,7 +12,11 @@ candid = { workspace = true }
 canister-test = { path = "../../../rust_canisters/canister_test" }
 chrono = { workspace = true }
 futures = { workspace = true }
+ic-agent = { workspace = true }
 ic-system-test-driver = { path = "../../driver" }
+ic-types = { path = "../../../types/types" }
+ic-universal-canister = { path = "../../../universal_canister/lib" }
+ic-utils = { workspace = true }
 slog = { workspace = true }
 tokio = { workspace = true }
 dfn_candid = { path = "../../../rust_canisters/dfn_candid" }