experiment: naive round-robin test splitting (compare vs #19423)

.circleci/config.yml

@@ -123,6 +123,7 @@ workflows:
             .* c-flake-shake-iterations << pipeline.parameters.flake-shake-iterations >> .circleci/continue/main.yml
             .* c-flake-shake-workers << pipeline.parameters.flake-shake-workers >> .circleci/continue/main.yml
             .* c-go-cache-version << pipeline.parameters.go-cache-version >> .circleci/continue/main.yml
+            rust/.* c-rust_files_changed true .circleci/continue/main.yml
 
             (rust|\.circleci)/.* c-rust_ci_dispatch << pipeline.parameters.rust_ci_dispatch >> .circleci/continue/rust-ci.yml
             (rust|\.circleci)/.* c-default_docker_image << pipeline.parameters.default_docker_image >> .circleci/continue/rust-ci.yml

.circleci/continue/main.yml

@@ -71,6 +71,11 @@ parameters:
   c-flake-shake-workers:
     type: integer
     default: 50
+  # Set to true by path-filtering when files under rust/ change.
+  # Used to skip expensive Rust builds on feature branches that don't touch Rust.
+  c-rust_files_changed:
+    type: boolean
+    default: false
   # go-cache-version can be used as a cache buster when making breaking changes to caching strategy
   c-go-cache-version:
     type: string
@@ -577,6 +582,10 @@ commands:
         description: "Whether to save the cache at the end of the build"
         type: boolean
         default: false
+      rust_files_changed:
+        description: "Whether Rust source files changed. When false on feature branches, the build is skipped if cached binaries are available."
+        type: boolean
+        default: true
     steps:
       - utils/checkout-with-mise:
           checkout-method: blobless
@@ -597,6 +606,25 @@ commands:
             PWD=$(pwd)
             export CARGO_TARGET_DIR="$PWD/<< parameters.directory >>/target"
             echo "CARGO_TARGET_DIR: $CARGO_TARGET_DIR"
+
+            # On feature branches where Rust files haven't changed, skip the expensive cargo build
+            # if the restored cache already contains the expected binaries. This saves ~9 minutes
+            # on PRs that don't touch Rust. Always build on develop/main as a safety backstop.
+            if [ "<< parameters.rust_files_changed >>" = "false" ] && \
+               [ "$CIRCLE_BRANCH" != "develop" ] && [ "$CIRCLE_BRANCH" != "main" ]; then
+              PROFILE_DIR="<< parameters.profile >>"
+              if [ "$PROFILE_DIR" = "debug" ]; then
+                PROFILE_DIR="debug"
+              fi
+              BINARY_DIR="$CARGO_TARGET_DIR/$PROFILE_DIR"
+              if ls "$BINARY_DIR"/kona-* "$BINARY_DIR"/op-* 2>/dev/null | head -1 > /dev/null; then
+                echo "Skipping Rust build: no Rust file changes on feature branch, using cached binaries"
+                ls -la "$BINARY_DIR"/kona-* "$BINARY_DIR"/op-* "$BINARY_DIR"/rollup-boost 2>/dev/null || true
+                exit 0
+              fi
+              echo "WARNING: No Rust changes detected but cached binaries not found. Building from source."
+            fi
+
             export PROFILE="--profile << parameters.profile >>"
 
             # Debug profile is specified as "debug" in the config/target, but cargo build expects "dev"
@@ -677,6 +705,10 @@ jobs:
         description: "Whether to persist the built binaries to the CircleCI workspace"
         type: boolean
         default: false
+      rust_files_changed:
+        description: "Whether Rust source files changed. When false on feature branches, the build is skipped if cached binaries are available."
+        type: boolean
+        default: true
     steps:
       - rust-build:
           directory: << parameters.directory >>
@@ -688,6 +720,7 @@ jobs:
           binary: << parameters.binary >>
           toolchain: << parameters.toolchain >>
           save_cache: << parameters.save_cache >>
+          rust_files_changed: << parameters.rust_files_changed >>
       - when:
           condition: << parameters.persist_to_workspace >>
           steps:
@@ -2023,6 +2056,10 @@ jobs:
         description: Timeout for when CircleCI kills the job if there's no output
         type: string
         default: 30m
+      skip_prepare:
+        description: Skip the 'go test -c' pre-compilation step. Set true for per-gate shard jobs where only a subset of packages are needed.
+        type: boolean
+        default: false
     docker:
       - image: <<pipeline.parameters.c-default_docker_image>>
     circleci_ip_ranges: true
@@ -2031,6 +2068,9 @@ jobs:
       - utils/checkout-with-mise:
           checkout-method: blobless
           enable-mise-cache: true
+      - run:
+          name: Verify shard coverage (no orphan test packages)
+          command: op-acceptance-tests/scripts/check-shard-coverage.sh
       - attach_workspace:
           at: .
       - run:
@@ -2057,11 +2097,15 @@ jobs:
           working_directory: op-acceptance-tests
           command: |
             just cmd-check
-      # Prepare the test environment
-      - run:
-          name: Prepare test environment (compile tests and cache build results)
-          working_directory: op-acceptance-tests
-          command: go test -v -c -o /dev/null $(go list -f '{{if .TestGoFiles}}{{.ImportPath}}{{end}}' ./tests/...)
+      # Prepare the test environment (skip for shard jobs where only a subset of packages are needed)
+      - when:
+          condition:
+            not: <<parameters.skip_prepare>>
+          steps:
+            - run:
+                name: Prepare test environment (compile tests and cache build results)
+                working_directory: op-acceptance-tests
+                command: go test -v -c -o /dev/null $(go list -f '{{if .TestGoFiles}}{{.ImportPath}}{{end}}' ./tests/...)
       # Run the acceptance tests (if the devnet is running)
       - run:
           name: Run acceptance tests (gate=<<parameters.gate>>)
@@ -2884,7 +2928,7 @@ workflows:
             branches:
               only: develop
       - contracts-bedrock-coverage:
-          # Generate coverage reports.
+          # Generate coverage reports. Runs on develop only — not on the PR critical path.
           name: contracts-bedrock-coverage <<matrix.features>>
           test_timeout: 1h
           test_profile: cicoverage
@@ -2895,6 +2939,9 @@ workflows:
           context:
             - circleci-repo-readonly-authenticated-github-token
             - slack
+          filters:
+            branches:
+              only: develop
       # On PRs, run upgrade tests with lite profile for better build times.
       - contracts-bedrock-tests-upgrade:
           name: contracts-bedrock-tests-upgrade op-mainnet <<matrix.features>>
@@ -2975,6 +3022,9 @@ workflows:
             - contracts-bedrock-build
           context:
             - circleci-repo-readonly-authenticated-github-token
+          filters:
+            branches:
+              only: develop
       - diff-fetcher-forge-artifacts:
           context:
             - circleci-repo-readonly-authenticated-github-token
@@ -3137,6 +3187,7 @@ workflows:
           features: "default"
           save_cache: true
           persist_to_workspace: true
+          rust_files_changed: << pipeline.parameters.c-rust_files_changed >>
           context:
             - circleci-repo-readonly-authenticated-github-token
       - rust-build-submodule: &rust-build-op-rbuilder
@@ -3162,11 +3213,25 @@ workflows:
       - go-binaries-for-sysgo:
           context:
             - circleci-repo-readonly-authenticated-github-token
-      # IN-MEMORY (all)
+      # IN-MEMORY (parallel shards) — replaces the single serial memory-all job.
+      # Each shard runs a non-overlapping subset of packages defined in acceptance-tests.yaml,
+      # so wall-clock time = longest shard, not sum of all tests.
       - op-acceptance-tests:
-          name: memory-all
-          gate: "" # Empty gate = gateless mode
-          no_output_timeout: 120m # Allow longer runs for memory-all gate
+          name: memory-shard-<<matrix.gate>>
+          gate: "ci-shard-<<matrix.gate>>"
+          no_output_timeout: 30m
+          skip_prepare: true
+          matrix:
+            parameters:
+              gate:
+                - "0"
+                - "1"
+                - "2"
+                - "3"
+                - "4"
+                - "5"
+                - "6"
+                - "7"
           context:
             - circleci-repo-readonly-authenticated-github-token
             - slack

Makefile

@@ -213,63 +213,80 @@ make-pre-test: ## Makes pre-test setup
 TEST_DEPS := op-program-client op-program-host cannon build-contracts cannon-prestates make-pre-test
 
 # Excludes: op-validator, op-deployer/pkg/{validation,deployer/{bootstrap,manage,opcm,pipeline,upgrade}} (need RPC)
+# Package list ordered for round-robin load balancing across 12 CI nodes.
+# Heavy packages are placed first so they spread across different nodes.
+# When reordering, ensure every package appears exactly once.
 TEST_PKGS := \
-	./op-alt-da/... \
-	./op-batcher/... \
-	./op-chain-ops/... \
+	./op-e2e/system/proofs \
+	./op-e2e/actions/proofs \
+	./op-e2e/actions/interop \
+	./op-e2e/system/bridge \
+	./op-e2e/system/da \
+	./op-e2e/interop/... \
+	./op-devstack/... \
+	./op-e2e/actions/batcher \
+	./op-supernode/... \
+	./op-service/... \
 	./op-node/... \
-	./op-proposer/... \
-	./op-challenger/... \
-	./op-faucet/... \
-	./op-dispute-mon/... \
-	./op-conductor/... \
+	./op-e2e/system/verifier \
+	./op-e2e/system/fees \
 	./op-program/... \
-	./op-service/... \
-	./op-supervisor/... \
-	./op-test-sequencer/... \
-	./op-fetcher/... \
-	./op-e2e/system/... \
-	./op-e2e/e2eutils/... \
+	./op-e2e/system/conductor \
+	./op-challenger/... \
 	./op-e2e/opgeth/... \
-	./op-e2e/interop/... \
-	./op-e2e/actions/altda \
-	./op-e2e/actions/batcher \
 	./op-e2e/actions/derivation \
+	./op-conductor/... \
+	./op-batcher/... \
+	./op-e2e/actions/sync \
+	./kurtosis-devnet/... \
+	./op-e2e/actions/upgrades \
 	./op-e2e/actions/helpers \
-	./op-e2e/actions/interop \
-	./op-e2e/actions/proofs \
+	./op-e2e/system/altda \
+	./op-e2e/system/contracts \
+	./op-e2e/system/e2esys \
+	./op-e2e/system/fjord \
+	./op-e2e/system/isthmus \
+	./op-e2e/system/p2p \
+	./op-e2e/system/runcfg \
+	./op-e2e/system/helpers \
+	./op-supervisor/... \
+	./op-e2e/actions/altda \
+	./op-e2e/actions/sequencer \
 	./op-e2e/actions/proposer \
 	./op-e2e/actions/safedb \
-	./op-e2e/actions/sequencer \
-	./op-e2e/actions/sync \
-	./op-e2e/actions/upgrades \
-	./packages/contracts-bedrock/scripts/checks/... \
-	./op-dripper/... \
-	./devnet-sdk/... \
-	./kurtosis-devnet/... \
-	./op-devstack/... \
+	./op-chain-ops/... \
+	./op-e2e/e2eutils/... \
 	./op-deployer/pkg/deployer/artifacts/... \
-	./op-deployer/pkg/deployer/broadcaster/... \
 	./op-deployer/pkg/deployer/clean/... \
-	./op-deployer/pkg/deployer/integration_test/ \
-	./op-deployer/pkg/deployer/integration_test/cli/... \
 	./op-deployer/pkg/deployer/standard/... \
 	./op-deployer/pkg/deployer/state/... \
 	./op-deployer/pkg/deployer/verify/... \
-	./op-sync-tester/... \
-	./op-supernode/...
+	./op-deployer/pkg/deployer/broadcaster/... \
+	./packages/contracts-bedrock/scripts/checks/... \
+	./op-alt-da/... \
+	./op-proposer/... \
+	./op-faucet/... \
+	./op-dispute-mon/... \
+	./op-dripper/... \
+	./op-test-sequencer/... \
+	./op-fetcher/... \
+	./devnet-sdk/... \
+	./op-sync-tester/...
 
 FRAUD_PROOF_TEST_PKGS := \
 	./op-e2e/faultproofs/...
 
-# Includes: op-validator, op-deployer/pkg/{bootstrap,manage,opcm,pipeline,upgrade} (need RPC)
+# RPC-dependent tests: run in go-tests-full (develop) only, not in go-tests-short (PR gate).
+# These fork live Ethereum state via Anvil and flake ~314 times/month under concurrent load.
 RPC_TEST_PKGS := \
 	./op-validator/pkg/validations/... \
 	./op-deployer/pkg/deployer/bootstrap/... \
 	./op-deployer/pkg/deployer/manage/... \
 	./op-deployer/pkg/deployer/opcm/... \
 	./op-deployer/pkg/deployer/pipeline/... \
-	./op-deployer/pkg/deployer/upgrade/...
+	./op-deployer/pkg/deployer/upgrade/... \
+	./op-deployer/pkg/deployer/integration_test/ \
+	./op-deployer/pkg/deployer/integration_test/cli/...
 
 # All test packages used by CI (combination of all package groups)
 ALL_TEST_PACKAGES := $(TEST_PKGS) $(RPC_TEST_PKGS) $(FRAUD_PROOF_TEST_PKGS)
@@ -307,7 +324,8 @@ go-tests-short: $(TEST_DEPS) ## Runs comprehensive Go tests with -short flag
 .PHONY: go-tests-short
 
 # Internal target for running Go tests with gotestsum for CI
-# Usage: make _go-tests-ci-internal GO_TEST_FLAGS="-short"
+# Usage: make _go-tests-ci-internal GO_TEST_FLAGS="-short" CI_TEST_PACKAGES="..."
+CI_TEST_PACKAGES ?= $(ALL_TEST_PACKAGES)
 _go-tests-ci-internal:
 	$(MAKE) -C cannon cannon elf # Required for cannon/provider_test TestLastStepCacheAccuracy
 	@echo "Setting up test directories..."
@@ -318,7 +336,7 @@ _go-tests-ci-internal:
 	if [ -n "$$CIRCLE_NODE_TOTAL" ] && [ "$$CIRCLE_NODE_TOTAL" -gt 1 ]; then \
 		export NODE_INDEX=$${CIRCLE_NODE_INDEX:-0} && \
 		export NODE_TOTAL=$${CIRCLE_NODE_TOTAL:-1} && \
-		export PARALLEL_PACKAGES=$$(echo "$(ALL_TEST_PACKAGES)" | tr ' ' '\n' | awk -v idx=$$NODE_INDEX -v total=$$NODE_TOTAL 'NR % total == idx' | tr '\n' ' ') && \
+		export PARALLEL_PACKAGES=$$(echo "$(CI_TEST_PACKAGES)" | tr ' ' '\n' | awk -v idx=$$NODE_INDEX -v total=$$NODE_TOTAL 'NR % total == idx' | tr '\n' ' ') && \
 		if [ -n "$$PARALLEL_PACKAGES" ]; then \
 			echo "Node $$NODE_INDEX/$$NODE_TOTAL running packages: $$PARALLEL_PACKAGES"; \
 			gotestsum --format=testname \
@@ -329,7 +347,7 @@ _go-tests-ci-internal:
 				--packages="$$PARALLEL_PACKAGES" \
 				-- -parallel=$$PARALLEL -coverprofile=coverage-$$NODE_INDEX.out $(GO_TEST_FLAGS) -timeout=$(TEST_TIMEOUT) -tags="ci"; \
 		else \
-			echo "ERROR: Node $$NODE_INDEX/$$NODE_TOTAL has no packages to run! Perhaps parallelism is set too high? (ALL_TEST_PACKAGES has $$(echo '$(ALL_TEST_PACKAGES)' | wc -w) packages)"; \
+			echo "ERROR: Node $$NODE_INDEX/$$NODE_TOTAL has no packages to run! Perhaps parallelism is set too high? (CI_TEST_PACKAGES has $$(echo '$(CI_TEST_PACKAGES)' | wc -w) packages)"; \
 			exit 1; \
 		fi; \
 	else \
@@ -338,13 +356,13 @@ _go-tests-ci-internal:
 			--jsonfile=./tmp/testlogs/log.json \
 			--rerun-fails=3 \
 			--rerun-fails-max-failures=50 \
-			--packages="$(ALL_TEST_PACKAGES)" \
+			--packages="$(CI_TEST_PACKAGES)" \
 			-- -parallel=$$PARALLEL -coverprofile=coverage.out $(GO_TEST_FLAGS) -timeout=$(TEST_TIMEOUT) -tags="ci"; \
 	fi
 .PHONY: _go-tests-ci-internal
 
 go-tests-short-ci: ## Runs short Go tests with gotestsum for CI (assumes deps built by CI)
-	$(MAKE) _go-tests-ci-internal GO_TEST_FLAGS="-short"
+	$(MAKE) _go-tests-ci-internal GO_TEST_FLAGS="-short" CI_TEST_PACKAGES="$(TEST_PKGS) $(FRAUD_PROOF_TEST_PKGS)"
 .PHONY: go-tests-short-ci
 
 go-tests-ci: ## Runs comprehensive Go tests with gotestsum for CI (assumes deps built by CI)

codecov.yml

@@ -38,7 +38,7 @@ coverage:
         base: auto
         target: auto
         threshold: 5%
-        informational: false
+        informational: true
         flags:
           - contracts-bedrock-tests
       # Kona patch coverage