diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f50831f548e..f088f1496bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ permissions: read-all jobs: vmss-virtual-a: - name: "VMSS Virtual A" # CI Checks, Clang Tidy, Python package tests, Doc build, Unit tests, Partitions tests + name: "VMSS Virtual A" # CI Checks, Clang Tidy, Python package tests, Doc build, Unit tests, e2e (bucket_a) runs-on: [ self-hosted, @@ -84,14 +84,11 @@ jobs: cd build ./tests.sh --output-on-failure -L unit -j$(nproc --all) - # Note that those are only run on the virtual CI, as they require enough - # privileges to configure iptables. ACI-based pools run unprivileged - # containers, and so cannot run these tests unfortunately. - - name: "Run partitions tests" + - name: "Run e2e tests (bucket_a)" run: | set -ex cd build - ./tests.sh --timeout 360 --output-on-failure -L partitions -C partitions + ./tests.sh --timeout 360 --output-on-failure -L bucket_a - name: "Upload logs for virtual A" uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -107,7 +104,7 @@ jobs: if: success() || failure() vmss-virtual-b: - name: "VMSS Virtual B" # End-to-end tests, except for partitions + name: "VMSS Virtual B" # End-to-end tests (bucket_b) runs-on: [ self-hosted, @@ -164,8 +161,7 @@ jobs: rm -rf /github/home/.cache mkdir -p /github/home/.cache - # End to end tests - ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|suite|unit" + ./tests.sh --timeout 360 --output-on-failure -L bucket_b shell: bash - name: "Upload logs for virtual B" @@ -182,12 +178,12 @@ jobs: if: success() || failure() vmss-virtual-c: - name: "VMSS Virtual C" # Code coverage + name: "VMSS Virtual C" # End-to-end tests (bucket_c) runs-on: [ self-hosted, 1ES.Pool=gha-vmss-d16av5-ci, - "JobId=ci_coverage-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}", + "JobId=ci_build_test_virtual_rest-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}", ] container: image: mcr.microsoft.com/azurelinux/base/core:3.0 @@ -212,64 +208,31 @@ jobs: set -ex ./scripts/setup-ci.sh - - name: "Build Debug with coverage" + - name: "Build Debug" run: | set -ex git config --global --add safe.directory /__w/CCF/CCF mkdir build cd build - cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DCOVERAGE=ON .. + cmake -GNinja -DCMAKE_BUILD_TYPE=Debug .. ninja shell: bash - - name: "Run Unit tests" - run: | - set -ex - cd build - ./tests.sh --output-on-failure -L unit -j$(nproc --all) - shell: bash - - - name: "End to end tests except partition" + - name: "Test virtual" run: | set -ex cd build rm -rf /github/home/.cache mkdir -p /github/home/.cache - # End to end tests - ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|suite|unit" + ./tests.sh --timeout 360 --output-on-failure -L bucket_c shell: bash - - name: "Generate coverage reports" - run: | - set -ex - cd build - ../scripts/coverage.sh --html coverage_html | tee coverage_report.txt - shell: bash - - - name: "Write coverage summary" - if: always() + - name: "Run partitions tests" run: | set -ex cd build - if [[ -f coverage_report.txt ]]; then - { - echo '## Code Coverage' - echo '' - echo '```' - cat coverage_report.txt - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - fi - shell: bash - - - name: "Upload HTML coverage report" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: coverage-report-html - path: build/coverage_html/ - if-no-files-found: ignore - if: success() || failure() + ./tests.sh --timeout 360 --output-on-failure -L partitions -C partitions - name: "Upload logs for virtual C" uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c3e797bd55..67e47b6eaeb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -923,6 +923,7 @@ if(BUILD_TESTS) ./raft_driver ${CMAKE_SOURCE_DIR}/tests/raft_scenarios/ ) set_property(TEST raft_scenario_test PROPERTY LABELS raft_scenario) + set_property(TEST raft_scenario_test APPEND PROPERTY LABELS bucket_c) add_test( NAME csr_test @@ -934,11 +935,13 @@ if(BUILD_TESTS) NAME versionifier_test COMMAND ${PYTHON} ${CMAKE_SOURCE_DIR}/python/src/ccf/_versionifier.py ) + set_property(TEST versionifier_test APPEND PROPERTY LABELS bucket_c) add_test( NAME github_version_lts_test COMMAND ${PYTHON} ${CMAKE_SOURCE_DIR}/tests/infra/github.py ) + set_property(TEST github_version_lts_test APPEND PROPERTY LABELS bucket_c) endif() if(NOT TSAN) @@ -983,6 +986,7 @@ if(BUILD_TESTS) add_e2e_test( NAME recovery_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/recovery.py + BUCKET bucket_b ADDITIONAL_ARGS ${ADDITIONAL_RECOVERY_ARGS} --constitution @@ -1051,6 +1055,7 @@ if(BUILD_TESTS) add_e2e_test( NAME js_batched_stress_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_batched.py + BUCKET bucket_c ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/src/apps/batched @@ -1062,6 +1067,7 @@ if(BUILD_TESTS) add_e2e_test( NAME modules_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/js-modules/modules.py + BUCKET bucket_c ADDITIONAL_ARGS --package js_generic @@ -1075,6 +1081,7 @@ if(BUILD_TESTS) NAME auth PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/js-custom-authorization/custom_authorization.py + BUCKET bucket_c ADDITIONAL_ARGS --package js_generic @@ -1100,6 +1107,7 @@ if(BUILD_TESTS) NAME governance_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/governance.py CONSTITUTION ${CONSTITUTION_ARGS} + BUCKET bucket_c ADDITIONAL_ARGS --initial-operator-count 1 @@ -1110,6 +1118,7 @@ if(BUILD_TESTS) add_e2e_test( NAME code_update_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/code_update.py + BUCKET bucket_c ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/logging/js @@ -1136,6 +1145,7 @@ if(BUILD_TESTS) add_e2e_test( NAME e2e_logging PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_logging.py + BUCKET bucket_c ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/logging/js ) @@ -1158,6 +1168,7 @@ if(BUILD_TESTS) CONSTITUTION ${RBAC_CONSTITUTION_ARGS} PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/programmability.py LABEL snp + BUCKET bucket_c ) # This test uses large requests (so too slow for SAN) @@ -1165,18 +1176,21 @@ if(BUILD_TESTS) add_e2e_test( NAME e2e_limits PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/limits.py + BUCKET bucket_c ) endif() add_e2e_test( NAME e2e_redirects PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/redirects.py + BUCKET bucket_c ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/logging/js ) add_e2e_test( NAME e2e_logging_http2 PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_logging.py + BUCKET bucket_c ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/logging/js @@ -1214,6 +1228,7 @@ if(BUILD_TESTS) add_e2e_test( NAME tls_stress_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/connections.py + BUCKET bucket_c ) if(CLIENT_PROTOCOLS_TEST) @@ -1221,12 +1236,14 @@ if(BUILD_TESTS) NAME client_protocols PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/client_protocols.py LABEL protocolstest + BUCKET bucket_c ) endif() add_e2e_test( NAME schema_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/schema.py + BUCKET bucket_b ADDITIONAL_ARGS --constitution ${CMAKE_SOURCE_DIR}/samples/constitutions/virtual/virtual_attestation_actions.js @@ -1258,6 +1275,7 @@ if(BUILD_TESTS) NAME lts_compatibility PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/lts_compatibility.py LABEL e2e + BUCKET bucket_a ADDITIONAL_ARGS ${LTS_TEST_ARGS} --constitution @@ -1283,6 +1301,7 @@ if(BUILD_TESTS) add_e2e_test( NAME nodes_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/nodes.py + BUCKET bucket_b ADDITIONAL_ARGS ${RECONFIG_TEST_ARGS} ${ROTATION_TEST_ARGS} ) @@ -1384,16 +1403,19 @@ if(BUILD_TESTS) add_e2e_test( NAME e2e_curl PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_curl.py + BUCKET bucket_c ) add_e2e_test( NAME historical_query_cache_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/historical_query_cache.py + BUCKET bucket_c ) add_e2e_test( NAME consistency_trace_validation PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/consistency_trace_validation.py + BUCKET bucket_c ) endif() diff --git a/cmake/common.cmake b/cmake/common.cmake index c52198d70c0..a8e619faef0 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -90,13 +90,18 @@ function(add_test_bin name) add_san(${name}) endfunction() -# Helper for building end-to-end function tests using the python infrastructure +# Helper for building end-to-end function tests using the python infrastructure. +# +# BUCKET assigns the test to one of the CI runner buckets (bucket_a, bucket_b, +# bucket_c) so that .github/workflows/ci.yml can select the per-runner test set +# with `ctest -L bucket_X`. Every PR-CI e2e test must be in exactly one bucket; +# scripts/test-buckets-checks.sh flags unbucketed tests in `no_bucket:`. function(add_e2e_test) cmake_parse_arguments( PARSE_ARGV 0 PARSED_ARGS "" - "NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT" + "NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT;BUCKET" "CONSTITUTION;ADDITIONAL_ARGS;CONFIGURATIONS" ) @@ -178,6 +183,14 @@ function(add_e2e_test) PROPERTY LABELS ${PARSED_ARGS_LABEL} ) + if(PARSED_ARGS_BUCKET) + set_property( + TEST ${PARSED_ARGS_NAME} + APPEND + PROPERTY LABELS ${PARSED_ARGS_BUCKET} + ) + endif() + if(${PARSED_ARGS_CURL_CLIENT}) set_property( TEST ${PARSED_ARGS_NAME} diff --git a/cmake/gersemi_definitions.cmake b/cmake/gersemi_definitions.cmake index 133ce90e8d8..68143b27293 100644 --- a/cmake/gersemi_definitions.cmake +++ b/cmake/gersemi_definitions.cmake @@ -24,7 +24,7 @@ function(add_e2e_test) PARSE_ARGV 0 PARSED_ARGS "" - "NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT" + "NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT;BUCKET" "CONSTITUTION;ADDITIONAL_ARGS;CONFIGURATIONS" ) endfunction() diff --git a/scripts/ci-checks.sh b/scripts/ci-checks.sh index c38e5081d50..a342bd775d7 100755 --- a/scripts/ci-checks.sh +++ b/scripts/ci-checks.sh @@ -48,6 +48,7 @@ CHECKS=( "Python format:python-format-checks.sh" "Python lint:python-lint-checks.sh" "Python types:python-types-checks.sh" + "CI test buckets:test-buckets-checks.sh" ) declare -A PID_TO_IDX diff --git a/scripts/test-buckets-checks.sh b/scripts/test-buckets-checks.sh new file mode 100755 index 00000000000..76a46d1fb8f --- /dev/null +++ b/scripts/test-buckets-checks.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +# Verifies that the e2e tests in each ctest CI runner bucket (bucket_a, +# bucket_b, bucket_c) match the frozen snapshot at tests/ci-buckets.txt. +# +# Catches: +# * an e2e test silently dropped from a bucket (regression of CI coverage), +# * an e2e test silently moved between buckets (load imbalance), +# * a new e2e test added without a BUCKET argument — it appears in the +# `no_bucket:` section instead of bucket_a/b/c. +# +# Unit tests are intentionally NOT bucketed: runner A selects them with +# `ctest -L unit`, so a new unit test can't be missed accidentally. Anything +# matching `-L unit` is excluded from the `no_bucket:` listing. +# +# Buckets are assigned via the BUCKET argument of add_e2e_test (see +# cmake/common.cmake) plus explicit set_property calls in CMakeLists.txt, +# and consumed by .github/workflows/ci.yml as `-L bucket_X`. +# +# The snapshot is captured for the default CMake configuration (no LONG_TESTS, +# no SAN, no COVERAGE, no CLIENT_PROTOCOLS_TEST) which is what the PR CI +# runners use. +# +# Usage: +# scripts/test-buckets-checks.sh # verify; fail with diff on mismatch +# scripts/test-buckets-checks.sh -f # regenerate the snapshot +# +# If a build/ directory is already configured with default flags, it is +# reused for speed; otherwise a fresh build directory is configured in +# $TMPDIR (which requires the project build dependencies to be installed). + +set -uo pipefail + +if [ "${1:-}" == "-f" ]; then + FIX=1 +else + FIX=0 +fi + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR=$( dirname "$SCRIPT_DIR" ) +SNAPSHOT="$ROOT_DIR/tests/ci-buckets.txt" + +cd "$ROOT_DIR" || exit 1 + +# Detect whether an existing build/ is usable. It must have the default flags +# (no LONG_TESTS, no SAN, no COVERAGE, no CLIENT_PROTOCOLS_TEST) so the +# inventory matches what PR CI sees. +build_dir_matches_defaults() { + local cache="$1/CMakeCache.txt" + [ -f "$cache" ] || return 1 + grep -q "^LONG_TESTS:BOOL=OFF" "$cache" || return 1 + grep -q "^SAN:BOOL=OFF" "$cache" || return 1 + grep -q "^COVERAGE:BOOL=OFF" "$cache" || return 1 + grep -q "^CLIENT_PROTOCOLS_TEST:BOOL=OFF" "$cache" || return 1 +} + +CLEANUP_BUILD="" +if build_dir_matches_defaults "$ROOT_DIR/build"; then + BUILD_DIR="$ROOT_DIR/build" + echo "Using existing build dir: $BUILD_DIR" +else + BUILD_DIR=$(mktemp -d) || { echo "mktemp failed" >&2; exit 1; } + CLEANUP_BUILD="$BUILD_DIR" + echo "Configuring fresh build dir for bucket check: $BUILD_DIR" + if ! cmake -GNinja -S "$ROOT_DIR" -B "$BUILD_DIR" -DCMAKE_BUILD_TYPE=Debug \ + >"$BUILD_DIR/configure.log" 2>&1; then + cat "$BUILD_DIR/configure.log" >&2 + echo "cmake configure failed; cannot run test bucket check" >&2 + rm -rf "$CLEANUP_BUILD" + exit 1 + fi +fi + +ACTUAL=$(mktemp) || { echo "mktemp failed" >&2; [ -n "$CLEANUP_BUILD" ] && rm -rf "$CLEANUP_BUILD"; exit 1; } +trap '[ -n "$CLEANUP_BUILD" ] && rm -rf "$CLEANUP_BUILD"; rm -f "$ACTUAL"' EXIT + +ctest_names() { + # Extract test names from `ctest -N` output. ctest -N lines look like + # " Test #12: foo_test", so we take the last whitespace-separated field. + (cd "$BUILD_DIR" && ctest -N "$@" 2>/dev/null) \ + | grep -E "^[[:space:]]*Test +#" \ + | awk '{print $NF}' \ + | sort -u +} + +emit_bucket() { + local bucket="$1" + echo "${bucket}:" + ctest_names -L "^${bucket}\$" | sed 's/^/ /' + echo +} + +# Tests that are neither labeled `unit` (handled by runner A's `-L unit`) nor +# placed in any bucket_* (handled by runners A/B/C's `-L bucket_X`). A new e2e +# test added without BUCKET will land here, making the regression obvious in +# the diff. Known-intentional residents are benchmarks (run in bencher +# workflows) and *_suite tests (run in long-test workflow). +emit_no_bucket() { + local all bucketed unit_tests + all=$(ctest_names) + bucketed=$(ctest_names -L 'bucket_') + unit_tests=$(ctest_names -L 'unit') + echo "no_bucket:" + comm -23 <(echo "$all") <(printf '%s\n%s\n' "$bucketed" "$unit_tests" | sort -u) \ + | sed 's/^/ /' + echo +} + +{ + cat <<'EOF' +# Frozen snapshot of CI e2e test bucket membership. Edited by +# scripts/test-buckets-checks.sh; do not hand-edit. +# +# Unit tests are not listed: they are selected with `ctest -L unit` and +# don't need explicit bucketing. Anything labeled `unit` is excluded from +# the no_bucket: section below. +# +# Buckets are assigned via add_e2e_test BUCKET (cmake/common.cmake) and +# consumed by .github/workflows/ci.yml as -L bucket_a/b/c. +# +# Snapshot is for the default CMake config (no LONG_TESTS, SAN, COVERAGE, +# CLIENT_PROTOCOLS_TEST). Tests gated by those flags do not appear here. +# +# To regenerate after an intentional change: +# ./scripts/test-buckets-checks.sh -f + +EOF + emit_bucket bucket_a + emit_bucket bucket_b + emit_bucket bucket_c + emit_no_bucket +} > "$ACTUAL" + +if [ "$FIX" -eq 1 ]; then + mv "$ACTUAL" "$SNAPSHOT" + # Recreate the trap target so EXIT cleanup doesn't fail on the moved file. + ACTUAL="" + trap '[ -n "$CLEANUP_BUILD" ] && rm -rf "$CLEANUP_BUILD"' EXIT + echo "Updated snapshot: $SNAPSHOT" + exit 0 +fi + +if [ ! -f "$SNAPSHOT" ]; then + echo "Snapshot file missing: $SNAPSHOT" >&2 + echo "Create one with: $0 -f" >&2 + exit 1 +fi + +if ! diff -u "$SNAPSHOT" "$ACTUAL"; then + cat <&2 + +ERROR: CI test bucket inventory diverged from $SNAPSHOT. + +Lines marked '-' are missing from (or moved out of) the current build; +lines marked '+' are new (or moved in). + +If the change is intentional, regenerate the snapshot: + $0 -f + +Common pitfall: a new add_e2e_test() call without a BUCKET argument will +appear under 'no_bucket:' instead of one of the e2e buckets. Add +'BUCKET bucket_a' (or bucket_b / bucket_c) to the call. +EOF + exit 1 +fi + +echo "Test buckets match snapshot ✓" diff --git a/tests/ci-buckets.txt b/tests/ci-buckets.txt new file mode 100644 index 00000000000..853617cdba6 --- /dev/null +++ b/tests/ci-buckets.txt @@ -0,0 +1,56 @@ +# Frozen snapshot of CI e2e test bucket membership. Edited by +# scripts/test-buckets-checks.sh; do not hand-edit. +# +# Unit tests are not listed: they are selected with `ctest -L unit` and +# don't need explicit bucketing. Anything labeled `unit` is excluded from +# the no_bucket: section below. +# +# Buckets are assigned via add_e2e_test BUCKET (cmake/common.cmake) and +# consumed by .github/workflows/ci.yml as -L bucket_a/b/c. +# +# Snapshot is for the default CMake config (no LONG_TESTS, SAN, COVERAGE, +# CLIENT_PROTOCOLS_TEST). Tests gated by those flags do not appear here. +# +# To regenerate after an intentional change: +# ./scripts/test-buckets-checks.sh -f + +bucket_a: + lts_compatibility + +bucket_b: + nodes_test + recovery_test + schema_test + +bucket_c: + auth + code_update_test + e2e_limits + e2e_logging + e2e_logging_http2 + e2e_redirects + github_version_lts_test + governance_test + js_batched_stress_test + modules_test + programmability_and_jwt + raft_scenario_test + tls_stress_test + versionifier_test + +no_bucket: + cose_bench + crypto_bench + full_test_suite + hash_bench + history_bench + json_bench + kv_bench + logger_bench + map_bench + merkle_bench + reconfiguration_test_suite + recovery_test_suite + ring_buffer_bench + task_bench +