diff --git a/.circleci/config.yml b/.circleci/config.yml index 73ff5875102aa..a9efa3d40b1c6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -123,6 +123,7 @@ workflows: .* c-flake-shake-iterations << pipeline.parameters.flake-shake-iterations >> .circleci/continue/main.yml .* c-flake-shake-workers << pipeline.parameters.flake-shake-workers >> .circleci/continue/main.yml .* c-go-cache-version << pipeline.parameters.go-cache-version >> .circleci/continue/main.yml + rust/.* c-rust_files_changed true .circleci/continue/main.yml (rust|\.circleci)/.* c-rust_ci_dispatch << pipeline.parameters.rust_ci_dispatch >> .circleci/continue/rust-ci.yml (rust|\.circleci)/.* c-default_docker_image << pipeline.parameters.default_docker_image >> .circleci/continue/rust-ci.yml diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index 6defd75af9b28..8fd308f81b495 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -71,6 +71,11 @@ parameters: c-flake-shake-workers: type: integer default: 50 + # Set to true by path-filtering when files under rust/ change. + # Used to skip expensive Rust builds on feature branches that don't touch Rust. + c-rust_files_changed: + type: boolean + default: false # go-cache-version can be used as a cache buster when making breaking changes to caching strategy c-go-cache-version: type: string @@ -577,6 +582,10 @@ commands: description: "Whether to save the cache at the end of the build" type: boolean default: false + rust_files_changed: + description: "Whether Rust source files changed. When false on feature branches, the build is skipped if cached binaries are available." + type: boolean + default: true steps: - utils/checkout-with-mise: checkout-method: blobless @@ -597,6 +606,25 @@ commands: PWD=$(pwd) export CARGO_TARGET_DIR="$PWD/<< parameters.directory >>/target" echo "CARGO_TARGET_DIR: $CARGO_TARGET_DIR" + + # On feature branches where Rust files haven't changed, skip the expensive cargo build + # if the restored cache already contains the expected binaries. This saves ~9 minutes + # on PRs that don't touch Rust. Always build on develop/main as a safety backstop. + if [ "<< parameters.rust_files_changed >>" = "false" ] && \ + [ "$CIRCLE_BRANCH" != "develop" ] && [ "$CIRCLE_BRANCH" != "main" ]; then + PROFILE_DIR="<< parameters.profile >>" + if [ "$PROFILE_DIR" = "debug" ]; then + PROFILE_DIR="debug" + fi + BINARY_DIR="$CARGO_TARGET_DIR/$PROFILE_DIR" + if ls "$BINARY_DIR"/kona-* "$BINARY_DIR"/op-* 2>/dev/null | head -1 > /dev/null; then + echo "Skipping Rust build: no Rust file changes on feature branch, using cached binaries" + ls -la "$BINARY_DIR"/kona-* "$BINARY_DIR"/op-* "$BINARY_DIR"/rollup-boost 2>/dev/null || true + exit 0 + fi + echo "WARNING: No Rust changes detected but cached binaries not found. Building from source." + fi + export PROFILE="--profile << parameters.profile >>" # Debug profile is specified as "debug" in the config/target, but cargo build expects "dev" @@ -677,6 +705,10 @@ jobs: description: "Whether to persist the built binaries to the CircleCI workspace" type: boolean default: false + rust_files_changed: + description: "Whether Rust source files changed. When false on feature branches, the build is skipped if cached binaries are available." + type: boolean + default: true steps: - rust-build: directory: << parameters.directory >> @@ -688,6 +720,7 @@ jobs: binary: << parameters.binary >> toolchain: << parameters.toolchain >> save_cache: << parameters.save_cache >> + rust_files_changed: << parameters.rust_files_changed >> - when: condition: << parameters.persist_to_workspace >> steps: @@ -2023,6 +2056,10 @@ jobs: description: Timeout for when CircleCI kills the job if there's no output type: string default: 30m + skip_prepare: + description: Skip the 'go test -c' pre-compilation step. Set true for per-gate shard jobs where only a subset of packages are needed. + type: boolean + default: false docker: - image: <> circleci_ip_ranges: true @@ -2031,6 +2068,9 @@ jobs: - utils/checkout-with-mise: checkout-method: blobless enable-mise-cache: true + - run: + name: Verify shard coverage (no orphan test packages) + command: op-acceptance-tests/scripts/check-shard-coverage.sh - attach_workspace: at: . - run: @@ -2057,11 +2097,15 @@ jobs: working_directory: op-acceptance-tests command: | just cmd-check - # Prepare the test environment - - run: - name: Prepare test environment (compile tests and cache build results) - working_directory: op-acceptance-tests - command: go test -v -c -o /dev/null $(go list -f '{{if .TestGoFiles}}{{.ImportPath}}{{end}}' ./tests/...) + # Prepare the test environment (skip for shard jobs where only a subset of packages are needed) + - when: + condition: + not: <> + steps: + - run: + name: Prepare test environment (compile tests and cache build results) + working_directory: op-acceptance-tests + command: go test -v -c -o /dev/null $(go list -f '{{if .TestGoFiles}}{{.ImportPath}}{{end}}' ./tests/...) # Run the acceptance tests (if the devnet is running) - run: name: Run acceptance tests (gate=<>) @@ -2884,7 +2928,7 @@ workflows: branches: only: develop - contracts-bedrock-coverage: - # Generate coverage reports. + # Generate coverage reports. Runs on develop only — not on the PR critical path. name: contracts-bedrock-coverage <> test_timeout: 1h test_profile: cicoverage @@ -2895,6 +2939,9 @@ workflows: context: - circleci-repo-readonly-authenticated-github-token - slack + filters: + branches: + only: develop # On PRs, run upgrade tests with lite profile for better build times. - contracts-bedrock-tests-upgrade: name: contracts-bedrock-tests-upgrade op-mainnet <> @@ -2975,6 +3022,9 @@ workflows: - contracts-bedrock-build context: - circleci-repo-readonly-authenticated-github-token + filters: + branches: + only: develop - diff-fetcher-forge-artifacts: context: - circleci-repo-readonly-authenticated-github-token @@ -3137,6 +3187,7 @@ workflows: features: "default" save_cache: true persist_to_workspace: true + rust_files_changed: << pipeline.parameters.c-rust_files_changed >> context: - circleci-repo-readonly-authenticated-github-token - rust-build-submodule: &rust-build-op-rbuilder @@ -3162,11 +3213,25 @@ workflows: - go-binaries-for-sysgo: context: - circleci-repo-readonly-authenticated-github-token - # IN-MEMORY (all) + # IN-MEMORY (parallel shards) — replaces the single serial memory-all job. + # Each shard runs a non-overlapping subset of packages defined in acceptance-tests.yaml, + # so wall-clock time = longest shard, not sum of all tests. - op-acceptance-tests: - name: memory-all - gate: "" # Empty gate = gateless mode - no_output_timeout: 120m # Allow longer runs for memory-all gate + name: memory-shard-<> + gate: "ci-shard-<>" + no_output_timeout: 30m + skip_prepare: true + matrix: + parameters: + gate: + - "0" + - "1" + - "2" + - "3" + - "4" + - "5" + - "6" + - "7" context: - circleci-repo-readonly-authenticated-github-token - slack diff --git a/Makefile b/Makefile index bfe1871860484..3ae5d722c3497 100644 --- a/Makefile +++ b/Makefile @@ -213,63 +213,80 @@ make-pre-test: ## Makes pre-test setup TEST_DEPS := op-program-client op-program-host cannon build-contracts cannon-prestates make-pre-test # Excludes: op-validator, op-deployer/pkg/{validation,deployer/{bootstrap,manage,opcm,pipeline,upgrade}} (need RPC) +# Package list ordered for round-robin load balancing across 12 CI nodes. +# Heavy packages are placed first so they spread across different nodes. +# When reordering, ensure every package appears exactly once. TEST_PKGS := \ - ./op-alt-da/... \ - ./op-batcher/... \ - ./op-chain-ops/... \ + ./op-e2e/system/proofs \ + ./op-e2e/actions/proofs \ + ./op-e2e/actions/interop \ + ./op-e2e/system/bridge \ + ./op-e2e/system/da \ + ./op-e2e/interop/... \ + ./op-devstack/... \ + ./op-e2e/actions/batcher \ + ./op-supernode/... \ + ./op-service/... \ ./op-node/... \ - ./op-proposer/... \ - ./op-challenger/... \ - ./op-faucet/... \ - ./op-dispute-mon/... \ - ./op-conductor/... \ + ./op-e2e/system/verifier \ + ./op-e2e/system/fees \ ./op-program/... \ - ./op-service/... \ - ./op-supervisor/... \ - ./op-test-sequencer/... \ - ./op-fetcher/... \ - ./op-e2e/system/... \ - ./op-e2e/e2eutils/... \ + ./op-e2e/system/conductor \ + ./op-challenger/... \ ./op-e2e/opgeth/... \ - ./op-e2e/interop/... \ - ./op-e2e/actions/altda \ - ./op-e2e/actions/batcher \ ./op-e2e/actions/derivation \ + ./op-conductor/... \ + ./op-batcher/... \ + ./op-e2e/actions/sync \ + ./kurtosis-devnet/... \ + ./op-e2e/actions/upgrades \ ./op-e2e/actions/helpers \ - ./op-e2e/actions/interop \ - ./op-e2e/actions/proofs \ + ./op-e2e/system/altda \ + ./op-e2e/system/contracts \ + ./op-e2e/system/e2esys \ + ./op-e2e/system/fjord \ + ./op-e2e/system/isthmus \ + ./op-e2e/system/p2p \ + ./op-e2e/system/runcfg \ + ./op-e2e/system/helpers \ + ./op-supervisor/... \ + ./op-e2e/actions/altda \ + ./op-e2e/actions/sequencer \ ./op-e2e/actions/proposer \ ./op-e2e/actions/safedb \ - ./op-e2e/actions/sequencer \ - ./op-e2e/actions/sync \ - ./op-e2e/actions/upgrades \ - ./packages/contracts-bedrock/scripts/checks/... \ - ./op-dripper/... \ - ./devnet-sdk/... \ - ./kurtosis-devnet/... \ - ./op-devstack/... \ + ./op-chain-ops/... \ + ./op-e2e/e2eutils/... \ ./op-deployer/pkg/deployer/artifacts/... \ - ./op-deployer/pkg/deployer/broadcaster/... \ ./op-deployer/pkg/deployer/clean/... \ - ./op-deployer/pkg/deployer/integration_test/ \ - ./op-deployer/pkg/deployer/integration_test/cli/... \ ./op-deployer/pkg/deployer/standard/... \ ./op-deployer/pkg/deployer/state/... \ ./op-deployer/pkg/deployer/verify/... \ - ./op-sync-tester/... \ - ./op-supernode/... + ./op-deployer/pkg/deployer/broadcaster/... \ + ./packages/contracts-bedrock/scripts/checks/... \ + ./op-alt-da/... \ + ./op-proposer/... \ + ./op-faucet/... \ + ./op-dispute-mon/... \ + ./op-dripper/... \ + ./op-test-sequencer/... \ + ./op-fetcher/... \ + ./devnet-sdk/... \ + ./op-sync-tester/... FRAUD_PROOF_TEST_PKGS := \ ./op-e2e/faultproofs/... -# Includes: op-validator, op-deployer/pkg/{bootstrap,manage,opcm,pipeline,upgrade} (need RPC) +# RPC-dependent tests: run in go-tests-full (develop) only, not in go-tests-short (PR gate). +# These fork live Ethereum state via Anvil and flake ~314 times/month under concurrent load. RPC_TEST_PKGS := \ ./op-validator/pkg/validations/... \ ./op-deployer/pkg/deployer/bootstrap/... \ ./op-deployer/pkg/deployer/manage/... \ ./op-deployer/pkg/deployer/opcm/... \ ./op-deployer/pkg/deployer/pipeline/... \ - ./op-deployer/pkg/deployer/upgrade/... + ./op-deployer/pkg/deployer/upgrade/... \ + ./op-deployer/pkg/deployer/integration_test/ \ + ./op-deployer/pkg/deployer/integration_test/cli/... # All test packages used by CI (combination of all package groups) ALL_TEST_PACKAGES := $(TEST_PKGS) $(RPC_TEST_PKGS) $(FRAUD_PROOF_TEST_PKGS) @@ -307,7 +324,8 @@ go-tests-short: $(TEST_DEPS) ## Runs comprehensive Go tests with -short flag .PHONY: go-tests-short # Internal target for running Go tests with gotestsum for CI -# Usage: make _go-tests-ci-internal GO_TEST_FLAGS="-short" +# Usage: make _go-tests-ci-internal GO_TEST_FLAGS="-short" CI_TEST_PACKAGES="..." +CI_TEST_PACKAGES ?= $(ALL_TEST_PACKAGES) _go-tests-ci-internal: $(MAKE) -C cannon cannon elf # Required for cannon/provider_test TestLastStepCacheAccuracy @echo "Setting up test directories..." @@ -318,7 +336,7 @@ _go-tests-ci-internal: if [ -n "$$CIRCLE_NODE_TOTAL" ] && [ "$$CIRCLE_NODE_TOTAL" -gt 1 ]; then \ export NODE_INDEX=$${CIRCLE_NODE_INDEX:-0} && \ export NODE_TOTAL=$${CIRCLE_NODE_TOTAL:-1} && \ - export PARALLEL_PACKAGES=$$(echo "$(ALL_TEST_PACKAGES)" | tr ' ' '\n' | awk -v idx=$$NODE_INDEX -v total=$$NODE_TOTAL 'NR % total == idx' | tr '\n' ' ') && \ + export PARALLEL_PACKAGES=$$(echo "$(CI_TEST_PACKAGES)" | tr ' ' '\n' | awk -v idx=$$NODE_INDEX -v total=$$NODE_TOTAL 'NR % total == idx' | tr '\n' ' ') && \ if [ -n "$$PARALLEL_PACKAGES" ]; then \ echo "Node $$NODE_INDEX/$$NODE_TOTAL running packages: $$PARALLEL_PACKAGES"; \ gotestsum --format=testname \ @@ -329,7 +347,7 @@ _go-tests-ci-internal: --packages="$$PARALLEL_PACKAGES" \ -- -parallel=$$PARALLEL -coverprofile=coverage-$$NODE_INDEX.out $(GO_TEST_FLAGS) -timeout=$(TEST_TIMEOUT) -tags="ci"; \ else \ - echo "ERROR: Node $$NODE_INDEX/$$NODE_TOTAL has no packages to run! Perhaps parallelism is set too high? (ALL_TEST_PACKAGES has $$(echo '$(ALL_TEST_PACKAGES)' | wc -w) packages)"; \ + echo "ERROR: Node $$NODE_INDEX/$$NODE_TOTAL has no packages to run! Perhaps parallelism is set too high? (CI_TEST_PACKAGES has $$(echo '$(CI_TEST_PACKAGES)' | wc -w) packages)"; \ exit 1; \ fi; \ else \ @@ -338,13 +356,13 @@ _go-tests-ci-internal: --jsonfile=./tmp/testlogs/log.json \ --rerun-fails=3 \ --rerun-fails-max-failures=50 \ - --packages="$(ALL_TEST_PACKAGES)" \ + --packages="$(CI_TEST_PACKAGES)" \ -- -parallel=$$PARALLEL -coverprofile=coverage.out $(GO_TEST_FLAGS) -timeout=$(TEST_TIMEOUT) -tags="ci"; \ fi .PHONY: _go-tests-ci-internal go-tests-short-ci: ## Runs short Go tests with gotestsum for CI (assumes deps built by CI) - $(MAKE) _go-tests-ci-internal GO_TEST_FLAGS="-short" + $(MAKE) _go-tests-ci-internal GO_TEST_FLAGS="-short" CI_TEST_PACKAGES="$(TEST_PKGS) $(FRAUD_PROOF_TEST_PKGS)" .PHONY: go-tests-short-ci go-tests-ci: ## Runs comprehensive Go tests with gotestsum for CI (assumes deps built by CI) diff --git a/codecov.yml b/codecov.yml index da49d85473dbd..abbf05c06790f 100644 --- a/codecov.yml +++ b/codecov.yml @@ -38,7 +38,7 @@ coverage: base: auto target: auto threshold: 5% - informational: false + informational: true flags: - contracts-bedrock-tests # Kona patch coverage diff --git a/op-acceptance-tests/acceptance-tests.yaml b/op-acceptance-tests/acceptance-tests.yaml index ead1240580fd8..df5c273612f86 100644 --- a/op-acceptance-tests/acceptance-tests.yaml +++ b/op-acceptance-tests/acceptance-tests.yaml @@ -176,6 +176,11 @@ gates: tests: - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync_tester/sync_tester_ext_el timeout: 30m + # sync_tester_hfs_ext: hardfork-switch external-network tests. Moved from PR CI + # (ci-shard-sync-b) to daily: too slow for PRs (800s), and covered by + # sync-a EL/CL sync tests for typical code changes. + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync_tester/sync_tester_hfs_ext + timeout: 30m - id: jovian inherits: @@ -204,3 +209,176 @@ gates: tests: - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/supernode/interop/... timeout: 15m + + # ============================================================================ + # CI PARALLEL SHARDS (NAIVE ROUND-ROBIN — infra#566 emulation) + # ============================================================================ + # Exact emulation of ethereum-optimism/infra#566's naive sharding: + # 1. Discover all 68 test packages on disk (loadGatelessValidators) + # 2. sort.Strings (Go lexicographic sort) + # 3. i % 8 == shardIndex (round-robin assignment) + # 4. --exclude-gates flake-shake (post-shard exclusion) + # + # This includes fault-proof tests, external-network tests, etc. — the naive + # approach has no awareness of test weight or prerequisites. + # ============================================================================ + + - id: ci-shard-0 + description: "Naive round-robin shard 0/8 (infra#566 emulation)" + tests: + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/batcher/throttling + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/ecotone + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/proofs + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/sync/multisupervisor_interop + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/isthmus/preinterop + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/safeheaddb_elsync + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync/clsync/gap_clp2p + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync_tester/sync_tester_elsync_multi + timeout: 20m + + - id: ci-shard-1 + description: "Naive round-robin shard 1/8 (infra#566 emulation)" + tests: + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base/chain + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/custom_gas_token + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/fjord + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/proofs-singlechain + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/sync/simple_interop + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/isthmus/preinterop-singlechain + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sequencer + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync/elsync/gap_clp2p + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync_tester/sync_tester_ext_el + timeout: 20m + + - id: ci-shard-2 + description: "Naive round-robin shard 2/8 (infra#566 emulation)" + tests: + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base/conductor + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/flashblocks + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/proofs/fpp + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/upgrade + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/isthmus/withdrawal_root + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/supernode + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync/elsync/gap_elp2p + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync_tester/sync_tester_hfs + timeout: 20m + + - id: ci-shard-3 + description: "Naive round-robin shard 3/8 (infra#566 emulation)" + tests: + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base/deposit + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/reqressyncdisabled/clsync + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/fusaka + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/proofs/serial + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/upgrade-no-supervisor + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/jovian/bpo2 + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync/elsync/reorg + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync_tester/sync_tester_hfs_ext + timeout: 20m + + - id: ci-shard-4 + description: "Naive round-robin shard 4/8 (infra#566 emulation)" + tests: + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base/withdrawal/cannon + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/reqressyncdisabled/divergence + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/contract + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/proofs/withdrawal + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/upgrade-singlechain + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/jovian/pectra + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync/follow_l2 + timeout: 20m + + - id: ci-shard-5 + description: "Naive round-robin shard 5/8 (infra#566 emulation)" + tests: + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base/withdrawal/cannon_kona + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/reqressyncdisabled/elsync + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/loadtest + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/reorgs + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/isthmus/erc20_bridge + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/proofs/cannon + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/supernode/interop/follow_l2 + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync/manual + timeout: 20m + + - id: ci-shard-6 + description: "Naive round-robin shard 6/8 (infra#566 emulation)" + tests: + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base/withdrawal/permissioned + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/syncmodereqressync/clsync + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/message + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/seqwindow + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/isthmus/operator_fee + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/rules + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/supernode/interop/reorg + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync_tester/sync_tester_e2e + timeout: 20m + + - id: ci-shard-7 + description: "Naive round-robin shard 7/8 (infra#566 emulation)" + tests: + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/batcher + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/prep + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/smoke + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/isthmus/pectra + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/safeheaddb_clsync + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/supernode/interop/same_timestamp_invalid + timeout: 20m + - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/sync_tester/sync_tester_elsync + timeout: 20m diff --git a/op-acceptance-tests/scripts/check-shard-coverage.sh b/op-acceptance-tests/scripts/check-shard-coverage.sh new file mode 100755 index 0000000000000..afc70f66bdc2c --- /dev/null +++ b/op-acceptance-tests/scripts/check-shard-coverage.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# check-shard-coverage.sh — Verify all test packages are covered by CI shard gates. +# +# Compares test packages on disk against the union of all ci-shard-* gates in +# acceptance-tests.yaml. Fails if any package with test files exists on disk +# but is not covered by a shard gate (or explicitly listed as excluded). +# +# Run from op-acceptance-tests/: +# ./scripts/check-shard-coverage.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +YAML="$ROOT_DIR/acceptance-tests.yaml" +TESTS_DIR="$ROOT_DIR/tests" + +MODULE="github.com/ethereum-optimism/optimism/op-acceptance-tests/tests" + +# --- Known exclusions (covered by develop-CI, nightly, or flake-shake) --- +# If you add a package here, add a comment explaining where it's covered. +EXCLUDED_PACKAGES=( + # Flake-shake quarantine — excluded by --exclude-gates flake-shake in infra#566 + "supernode/interop/activation" + "depreqres/syncmodereqressync/elsync" + "depreqres/reqressyncdisabled" + "supernode/interop" +) + +# --- Find all test packages on disk that contain actual test functions --- +disk_packages=() +while IFS= read -r dir; do + # Only count packages with actual Test functions (not just _test.go boilerplate) + if grep -rql '^func Test' "$dir"/*_test.go 2>/dev/null; then + rel="${dir#"$TESTS_DIR"/}" + disk_packages+=("$rel") + fi +done < <(find "$TESTS_DIR" -name '*_test.go' -printf '%h\n' | sort -u) + +# --- Extract packages from ci-shard-* gates in acceptance-tests.yaml --- +# Handles both exact packages and .../... wildcard packages. +shard_packages=() +in_shard=false +while IFS= read -r line; do + if [[ "$line" =~ id:\ *ci-shard- ]]; then + in_shard=true + continue + fi + if $in_shard && [[ "$line" =~ ^[[:space:]]*-\ *id: ]]; then + in_shard=false + continue + fi + if $in_shard && [[ "$line" =~ package:.*$MODULE/ ]]; then + pkg="${line##*"$MODULE"/}" + pkg="${pkg%%\"*}" + pkg="${pkg%%\'*}" + pkg="$(echo "$pkg" | xargs)" # trim whitespace + shard_packages+=("$pkg") + fi +done < "$YAML" + +# --- Check coverage --- +missing=() +for disk_pkg in "${disk_packages[@]}"; do + covered=false + + # Check explicit exclusions + for excl in "${EXCLUDED_PACKAGES[@]}"; do + if [[ "$disk_pkg" == "$excl" ]]; then + covered=true + break + fi + done + $covered && continue + + # Check shard gates + for shard_pkg in "${shard_packages[@]}"; do + # Exact match + if [[ "$disk_pkg" == "$shard_pkg" ]]; then + covered=true + break + fi + # Wildcard match: "foo/..." covers "foo", "foo/bar", "foo/bar/baz" + if [[ "$shard_pkg" == *"/..." ]]; then + prefix="${shard_pkg%/...}" + if [[ "$disk_pkg" == "$prefix" || "$disk_pkg" == "$prefix/"* ]]; then + covered=true + break + fi + fi + done + + if ! $covered; then + missing+=("$disk_pkg") + fi +done + +if [[ ${#missing[@]} -gt 0 ]]; then + echo "ERROR: The following test packages are not covered by any ci-shard-* gate" + echo " and are not in the exclusion list:" + echo "" + for pkg in "${missing[@]}"; do + echo " - tests/$pkg" + done + echo "" + echo "To fix: add the package to a ci-shard-* gate in acceptance-tests.yaml," + echo " or add it to EXCLUDED_PACKAGES in this script with a comment" + echo " explaining where it's covered (develop-CI, nightly, flake-shake)." + exit 1 +fi + +echo "OK: All ${#disk_packages[@]} test packages are covered by ci-shard gates (${#shard_packages[@]} shard entries, ${#EXCLUDED_PACKAGES[@]} exclusions)."