chore: remove flaky settings since flaky=True is now the default for all tests (#9016)

basvandijk · web-flow · commit 89cc1c202235 · 2026-02-27T22:12:33.000Z
Since 6cee23b all tests are automatically retried on failure (up to 3 times). When a test attempt passes after having failed in a previous attempt the test run will be considered flaky. This allows us to detect and fix flakiness ASAP without it being hidden. This means the `flaky = True` settings are no longer necessary and can be removed.
diff --git a/bazel/README.md b/bazel/README.md
@@ -283,18 +283,6 @@ Bazel provides several tools to **mitigate** and **resolve** flaky tests.
 
 ## Mitigation
 
-Mark the test as **flaky** to make Bazel will retry the test up to three times.
-
-```bash
-rust_test(
-	name = "foo_test",
-  # lines omitted
-	flaky = True",  # flakiness rate of $f% over the last month on $date.
-)
-```
-
-Where you can retrieve the flakiness rate $f from Superset.
-
 Instruct rust to only run one test in parallel - this can help when multiple
 concurrent test cases collide but may greatly increase the runtime of the tests.
 
diff --git a/packages/pocket-ic/BUILD.bazel b/packages/pocket-ic/BUILD.bazel
@@ -70,7 +70,6 @@ rust_test(
         "SSL_CERT_FILE": "$(rootpath @mozilla_root_ca_store//file)",
         "TEST_WASM": "$(rootpath //packages/pocket-ic/test_canister:test_canister.wasm.gz)",
     },
-    flaky = True,
     tags = [
         "cpu:16",
         # TODO: remove 'requires-network' tag when the root cause for sporadic error below on Apple Silicon is identified and fixed.
@@ -125,7 +124,6 @@ rust_test(
         "SSL_CERT_FILE": "$(rootpath @mozilla_root_ca_store//file)",
         "TEST_WASM": "$(rootpath //packages/pocket-ic/test_canister:test_canister.wasm.gz)",
     },
-    flaky = True,
     tags = [
         "cpu:16",
         # TODO: remove 'requires-network' tag when the root cause for sporadic error below on Apple Silicon is identified and fixed.
diff --git a/rs/bitcoin/adapter/BUILD.bazel b/rs/bitcoin/adapter/BUILD.bazel
@@ -137,7 +137,6 @@ rust_test(
         "RUST_TEST_THREADS": "4",
         "RUST_TEST_NOCAPTURE": "1",
     },
-    flaky = True,
     tags = [
         "cpu:4",
     ],
diff --git a/rs/nns/integration_tests/BUILD.bazel b/rs/nns/integration_tests/BUILD.bazel
@@ -279,7 +279,6 @@ rust_ic_test_suite_with_extra_srcs(
         "JAVABASE": "$(JAVABASE)",
     },
     extra_srcs = ["src/lib.rs"],
-    flaky = True,  # flakiness rate of 1.29% over the month from 2025-02-11 till 2025-03-11 for the "governance_time_warp" test
     proc_macro_deps = MACRO_DEPENDENCIES + MACRO_DEV_DEPENDENCIES,
     toolchains = ["@bazel_tools//tools/jdk:current_java_runtime"],  # for TLA/Apalache
     deps = DEPENDENCIES_WITH_TEST_FEATURES + DEV_DEPENDENCIES,
diff --git a/rs/rosetta-api/icp/tests/integration_tests/BUILD.bazel b/rs/rosetta-api/icp/tests/integration_tests/BUILD.bazel
@@ -66,7 +66,6 @@ rust_test_suite(
     },
     # The test is critical to get resources timely and therefore fails sometimes when run on heavily loaded node.
     # TODO(IDX-2225): reconsider when we will use Remote Execution.
-    flaky = True,  # flakiness rate of over 2% over the month from 2025-02-11 till 2025-03-11.
     proc_macro_deps = [
     ],
     tags = ["cpu:4"],
diff --git a/rs/rosetta-api/icrc1/BUILD.bazel b/rs/rosetta-api/icrc1/BUILD.bazel
@@ -180,7 +180,6 @@ rust_test_suite_with_extra_srcs(
     extra_srcs = glob([
         "tests/common/*.rs",
     ]),
-    flaky = True,
     proc_macro_deps = MACRO_DEV_DEPENDENCIES,
     deps = DEV_DEPENDENCIES + DEPENDENCIES,
 )
@@ -204,7 +203,6 @@ rust_test_suite_with_extra_srcs(
     extra_srcs = glob([
         "tests/common/*.rs",
     ]),
-    flaky = True,
     proc_macro_deps = MACRO_DEV_DEPENDENCIES,
     tags = ["cpu:4"],
     deps = DEV_DEPENDENCIES + DEPENDENCIES,
diff --git a/rs/sns/integration_tests/BUILD.bazel b/rs/sns/integration_tests/BUILD.bazel
@@ -212,7 +212,6 @@ rust_ic_test_suite_with_extra_srcs(
     data = DATA_DEPS,
     env = ENV,
     extra_srcs = [],
-    flaky = True,  # flakiness rate of over 1% for the "proposals" and "upgrade_canister" tests over the month from 2025-02-11 till 2025-03-11.
     proc_macro_deps = MACRO_DEPENDENCIES + MACRO_DEV_DEPENDENCIES,
     tags = ["cpu:8"],
     deps = DEPENDENCIES_WITH_TEST_FEATURES + TEST_DEV_DEPENDENCIES,
@@ -229,7 +228,6 @@ rust_ic_test(
     crate_features = ["test"],
     data = DATA_DEPS,
     env = ENV,
-    flaky = True,  # flakiness rate of over 1% over the month from 2025-02-11 till 2025-03-11.
     #proc_macro_deps = MACRO_DEPENDENCIES + MACRO_DEV_DEPENDENCIES,
     tags = [
         "cpu:8",
diff --git a/rs/tests/consensus/orchestrator/BUILD.bazel b/rs/tests/consensus/orchestrator/BUILD.bazel
@@ -103,7 +103,6 @@ system_test_nns(
 
 system_test_nns(
     name = "unstuck_subnet_test",
-    flaky = True,  # flakiness rate of over 2.1% over the month from 2025-02-11 till 2025-03-11.
     guestos_update = "test",
     tags = [
         "long_test",
diff --git a/rs/tests/consensus/subnet_recovery/BUILD.bazel b/rs/tests/consensus/subnet_recovery/BUILD.bazel
@@ -76,7 +76,6 @@ system_test_nns(
 
 system_test_nns(
     name = "sr_app_same_nodes_enable_chain_keys_test",
-    flaky = True,  # flakiness rate of over 3.33% over the month from 2025-02-11 till 2025-03-11.
     guestos_update = "test",
     tags = [
         "colocate",
@@ -128,7 +127,6 @@ system_test_nns(
 
 system_test_nns(
     name = "sr_app_failover_nodes_enable_chain_keys_test",
-    flaky = True,  # flakiness rate of over 1.67% over the month from 2025-02-11 till 2025-03-11.
     guestos_update = "test",
     tags = [
         "colocate",
@@ -146,7 +144,6 @@ system_test_nns(
 
 system_test_nns(
     name = "sr_app_failover_nodes_with_chain_keys_test",
-    flaky = True,  # flakiness rate of 3.33% over the month from 2025-02-11 till 2025-03-11.
     guestos_update = "test",
     tags = [
         "colocate",
@@ -215,7 +212,6 @@ system_test_nns(
 
 system_test_nns(
     name = "sr_app_no_upgrade_with_chain_keys_test",
-    flaky = True,  # flakiness rate of over 6% over the month from 2025-02-11 till 2025-03-11 (only for //rs/tests/consensus/subnet_recovery:sr_app_no_upgrade_with_chain_keys_test_head_nns_colocate).
     guestos_update = "test",
     tags = [
         "colocate",
@@ -234,7 +230,6 @@ system_test_nns(
 system_test_nns(
     name = "sr_app_large_no_upgrade_with_chain_keys_test",
     enable_head_nns_variant = False,  # Let's not run this expensive test against the HEAD NNS canisters to save resources.
-    flaky = True,  # flakiness rate of over 2.21% over the month from 2025-02-11 till 2025-03-11.
     guestos_update = "test",
     tags = [
         "colocate",
diff --git a/rs/tests/consensus/tecdsa/BUILD.bazel b/rs/tests/consensus/tecdsa/BUILD.bazel
@@ -132,7 +132,6 @@ system_test_nns(
 
 system_test_nns(
     name = "tecdsa_signature_fails_without_cycles_test",
-    flaky = True,  # flakiness rate of over 1.05% over the month from 2025-02-11 till 2025-03-11.
     tags = [
         "long_test",
     ],
@@ -154,7 +153,6 @@ system_test_nns(
 
 system_test_nns(
     name = "tecdsa_signature_from_nns_without_cycles_test",
-    flaky = True,  # flakiness rate of over 1.67% over the month from 2025-02-11 till 2025-03-11 only for the //rs/tests/consensus/tecdsa:tecdsa_signature_from_nns_without_cycles_test_head_nns variant.
     tags = [
         "long_test",
     ],
@@ -215,7 +213,6 @@ system_test_nns(
 system_test_nns(
     name = "tecdsa_signature_life_cycle_test",
     # Remove when CON-937 is resolved
-    flaky = True,  # flakiness rate of 1.57% over the month from 2025-02-11 till 2025-03-11
     tags = [
         "long_test",  # since it takes longer than 5 minutes.
     ],
diff --git a/rs/tests/consensus/upgrade/BUILD.bazel b/rs/tests/consensus/upgrade/BUILD.bazel
@@ -102,7 +102,6 @@ rust_binary(
 system_test_nns(
     name = "upgrade_app_subnet_test",
     env = MAINNET_ENV,
-    flaky = True,  # flakiness rate of over 2.27% over the month from 2025-02-11 till 2025-03-11.
     guestos = "mainnet_nns",
     guestos_update = True,
     tags = [
@@ -116,7 +115,6 @@ system_test_nns(
 system_test_nns(
     name = "downgrade_app_subnet_test",
     env = MAINNET_ENV,
-    flaky = True,  # flakiness rate of over 2.97% over the month from 2025-02-11 till 2025-03-11.
     guestos_update = "mainnet_nns",
     tags = [
         "long_test",  # since it takes longer than 5 minutes.
diff --git a/rs/tests/cross_chain/BUILD.bazel b/rs/tests/cross_chain/BUILD.bazel
@@ -107,7 +107,6 @@ oci_tar(
 
 system_test_nns(
     name = "btc_get_balance_test",
-    flaky = True,  # flakiness rate of over 3% over the month from 2025-02-11 till 2025-03-11 only for the btc_get_balance_test_head_nns variant.
     tags = [
         "colocate",
     ],
diff --git a/rs/tests/nested/BUILD.bazel b/rs/tests/nested/BUILD.bazel
@@ -50,7 +50,6 @@ genrule(
 system_test_nns(
     name = "registration",
     env = MAINNET_ENV,
-    flaky = True,  # flakiness rate of over 2% over the month from 2025-02-11 till 2025-03-11.
     setupos = True,
     tags = ["long_test"],  # since it takes longer than 5 minutes.
     test_timeout = "eternal",
@@ -82,7 +81,6 @@ rust_binary(
 system_test_nns(
     name = "guestos_upgrade_smoke_test",
     env = MAINNET_ENV,
-    flaky = True,
     guestos_update = "test",
     setupos = True,
     tags = ["long_test"],
@@ -94,7 +92,6 @@ system_test_nns(
 # Tests if "latest_release" in mainnet-icos-revisions.json can successfully guestos upgrade to current commit
 system_test_nns(
     name = "guestos_upgrade_from_latest_release_to_current",
-    flaky = True,
     guestos = "mainnet_latest_dev",  # necessary for configuring NNS public key
     guestos_update = True,
     setupos = "mainnet_latest_dev",
@@ -146,7 +143,6 @@ rust_binary(
 system_test_nns(
     name = "hostos_upgrade_smoke_test",
     env = MAINNET_ENV,
-    flaky = True,
     hostos_update = "test",
     setupos = True,
     tags = ["long_test"],
@@ -158,7 +154,6 @@ system_test_nns(
 # Tests if "latest_release" in mainnet-icos-revisions.json can successfully hostos upgrade to current commit
 system_test_nns(
     name = "hostos_upgrade_from_latest_release_to_current",
-    flaky = True,  # flakiness rate of 5% over the month from 2025-02-11 till 2025-03-11.
     guestos = "mainnet_latest_dev",  # necessary for configuring NNS public key
     hostos_update = True,
     setupos = "mainnet_latest_dev",
diff --git a/rs/tests/nested/nns_recovery/BUILD.bazel b/rs/tests/nested/nns_recovery/BUILD.bazel
@@ -42,7 +42,6 @@ rust_library(
 system_test_nns(
     name = "nr_broken_dfinity_node",
     env = ENV,
-    flaky = True,
     guestos_update = "test",
     setupos = True,
     tags = ["long_test"],
@@ -60,7 +59,6 @@ system_test_nns(
     name = "nr_large",
     enable_head_nns_variant = False,
     env = ENV,
-    flaky = True,
     guestos_update = "test",
     setupos = True,
     tags = [
@@ -79,7 +77,6 @@ system_test_nns(
 system_test_nns(
     name = "nr_local",
     env = ENV,
-    flaky = True,
     guestos_update = "test",
     setupos = True,
     tags = ["long_test"],
@@ -96,7 +93,6 @@ system_test_nns(
 system_test_nns(
     name = "nr_no_bless_fix_like_np",
     env = ENV,
-    flaky = True,
     guestos_update = "test",
     setupos = True,
     tags = ["long_test"],
@@ -113,7 +109,6 @@ system_test_nns(
 system_test_nns(
     name = "nr_seq_np_actions",
     env = ENV,
-    flaky = True,
     guestos_update = "test",
     setupos = True,
     tags = ["long_test"],
diff --git a/rs/tests/networking/BUILD.bazel b/rs/tests/networking/BUILD.bazel
@@ -279,7 +279,6 @@ system_test_nns(
 
 system_test_nns(
     name = "firewall_max_connections_test",
-    flaky = True,  # flakiness rate of 5% over the month from 2025-02-11 till 2025-03-11.
     tags = [
         "colocate",
         "long_test",
@@ -308,7 +307,6 @@ system_test_nns(
 
 system_test_nns(
     name = "http_endpoints_public_spec_test",
-    flaky = False,
     runtime_deps = UNIVERSAL_CANISTER_RUNTIME_DEPS,
     deps = COMMON_DEPS + HTTP_ENDPOINTS_DEPS + [
         "@crate_index//:futures",
@@ -321,7 +319,6 @@ system_test_nns(
 system_test_nns(
     name = "network_large_test",
     enable_head_nns_variant = False,  # only run this test with the mainnet NNS canisters.
-    flaky = True,  # flakiness rate of 20% over the month from 2025-02-11 till 2025-03-11.
     tags = [
         "system_test_large",
     ],
@@ -338,7 +335,6 @@ system_test_nns(
 system_test_nns(
     name = "network_reliability_test",
     enable_head_nns_variant = False,  # only run this test with the mainnet NNS canisters.
-    flaky = True,  # flakiness rate of 3.45% over the month from 2025-02-11 till 2025-03-11.
     tags = [
         "colocate",
         "system_test_large",
@@ -378,7 +374,6 @@ system_test_nns(
     },
     enable_head_nns_variant = False,
     enable_metrics = True,
-    flaky = True,  # flakiness rate of over 1.1% over the month from 2025-02-11 till 2025-03-11.
     prometheus_vm_required_host_features = ["performance"],
     tags = [
         "colocate",
@@ -428,7 +423,6 @@ system_test_nns(
     },
     enable_head_nns_variant = False,
     enable_metrics = True,
-    flaky = True,  # flakiness rate of 1.83% over the month from 2025-02-11 till 2025-03-11.
     prometheus_vm_required_host_features = ["performance"],
     tags = [
         "colocate",
diff --git a/rs/tests/query_stats/BUILD.bazel b/rs/tests/query_stats/BUILD.bazel
@@ -18,7 +18,6 @@ system_test(
 
 system_test(
     name = "query_stats_below_threshold",
-    flaky = True,  # flakiness rate of 1.22% over the month from 2025-02-11 till 2025-03-11.
     tags = [
         "long_test",  # since it takes longer than 5 minutes.
     ],
diff --git a/rs/tests/system_tests.bzl b/rs/tests/system_tests.bzl
@@ -22,7 +22,6 @@ def system_test(
         runtime_deps = {},
         tags = [],
         test_timeout = "long",
-        flaky = False,
         enable_metrics = False,
         prometheus_vm_required_host_features = [],
         prometheus_vm_resources = default_vm_resources,
@@ -52,7 +51,6 @@ def system_test(
       runtime_deps: dependencies to make available to the test when it runs.
       tags: additional tags for the system_test.
       test_timeout: bazel test timeout (short, moderate, long or eternal).
-      flaky: rerun in case of failure (up to 3 times).
       enable_metrics: if True, a PrometheusVm will be spawned running both p8s (configured to scrape the testnet) & Grafana.
       prometheus_vm_required_host_features: a list of strings specifying the required host features of the PrometheusVm.
       prometheus_vm_resources: a structure describing the required resources of the PrometheusVm. For example:
@@ -224,7 +222,6 @@ def system_test(
         tags = tags + (["manual"] if "colocate" in tags else []),
         target_compatible_with = ["@platforms//os:linux"],
         timeout = test_timeout,
-        flaky = flaky,
         visibility = visibility,
     )
 
@@ -252,7 +249,6 @@ def system_test(
         tags = tags + (["manual"] if not "colocate" in tags else []) + additional_colocate_tags,
         target_compatible_with = ["@platforms//os:linux"],
         timeout = test_timeout,
-        flaky = flaky,
         visibility = visibility,
     )
     return struct(test_driver_target = test_driver_target)
