CI tests reorg & coverage drop

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ permissions: read-all
 
 jobs:
   vmss-virtual-a:
-    name: "VMSS Virtual A" # CI Checks, Clang Tidy, Python package tests, Doc build, Unit tests, Partitions tests
+    name: "VMSS Virtual A" # CI Checks, Clang Tidy, Python package tests, Doc build, Unit tests, e2e (bucket_a)
     runs-on:
       [
         self-hosted,
@@ -84,14 +84,11 @@ jobs:
           cd build
           ./tests.sh --output-on-failure -L unit -j$(nproc --all)
 
-        # Note that those are only run on the virtual CI, as they require enough
-        # privileges to configure iptables. ACI-based pools run unprivileged
-        # containers, and so cannot run these tests unfortunately.
-      - name: "Run partitions tests"
+      - name: "Run e2e tests (bucket_a)"
         run: |
           set -ex
           cd build
-          ./tests.sh --timeout 360 --output-on-failure -L partitions -C partitions
+          ./tests.sh --timeout 360 --output-on-failure -L bucket_a
 
       - name: "Upload logs for virtual A"
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
@@ -107,7 +104,7 @@ jobs:
         if: success() || failure()
 
   vmss-virtual-b:
-    name: "VMSS Virtual B" # End-to-end tests, except for partitions
+    name: "VMSS Virtual B" # End-to-end tests (bucket_b)
     runs-on:
       [
         self-hosted,
@@ -164,8 +161,7 @@ jobs:
           rm -rf /github/home/.cache
           mkdir -p /github/home/.cache
 
-          # End to end tests
-          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|suite|unit"
+          ./tests.sh --timeout 360 --output-on-failure -L bucket_b
         shell: bash
 
       - name: "Upload logs for virtual B"
@@ -182,12 +178,12 @@ jobs:
         if: success() || failure()
 
   vmss-virtual-c:
-    name: "VMSS Virtual C" # Code coverage
+    name: "VMSS Virtual C" # End-to-end tests (bucket_c)
     runs-on:
       [
         self-hosted,
         1ES.Pool=gha-vmss-d16av5-ci,
-        "JobId=ci_coverage-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
+        "JobId=ci_build_test_virtual_rest-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
       ]
     container:
       image: mcr.microsoft.com/azurelinux/base/core:3.0
@@ -212,64 +208,31 @@ jobs:
           set -ex
           ./scripts/setup-ci.sh
 
-      - name: "Build Debug with coverage"
+      - name: "Build Debug"
         run: |
           set -ex
           git config --global --add safe.directory /__w/CCF/CCF
           mkdir build
           cd build
-          cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DCOVERAGE=ON ..
+          cmake -GNinja -DCMAKE_BUILD_TYPE=Debug ..
           ninja
         shell: bash
 
-      - name: "Run Unit tests"
-        run: |
-          set -ex
-          cd build
-          ./tests.sh --output-on-failure -L unit -j$(nproc --all)
-        shell: bash
-
-      - name: "End to end tests except partition"
+      - name: "Test virtual"
         run: |
           set -ex
           cd build
           rm -rf /github/home/.cache
           mkdir -p /github/home/.cache
 
-          # End to end tests
-          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|suite|unit"
+          ./tests.sh --timeout 360 --output-on-failure -L bucket_c
         shell: bash
 
-      - name: "Generate coverage reports"
-        run: |
-          set -ex
-          cd build
-          ../scripts/coverage.sh --html coverage_html | tee coverage_report.txt
-        shell: bash
-
-      - name: "Write coverage summary"
-        if: always()
+      - name: "Run partitions tests"
         run: |
           set -ex
           cd build
-          if [[ -f coverage_report.txt ]]; then
-            {
-              echo '## Code Coverage'
-              echo ''
-              echo '```'
-              cat coverage_report.txt
-              echo '```'
-            } >> "$GITHUB_STEP_SUMMARY"
-          fi
-        shell: bash
-
-      - name: "Upload HTML coverage report"
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
-        with:
-          name: coverage-report-html
-          path: build/coverage_html/
-          if-no-files-found: ignore
-        if: success() || failure()
+          ./tests.sh --timeout 360 --output-on-failure -L partitions -C partitions
 
       - name: "Upload logs for virtual C"
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7

CMakeLists.txt

@@ -923,6 +923,7 @@ if(BUILD_TESTS)
         ./raft_driver ${CMAKE_SOURCE_DIR}/tests/raft_scenarios/
     )
     set_property(TEST raft_scenario_test PROPERTY LABELS raft_scenario)
+    set_property(TEST raft_scenario_test APPEND PROPERTY LABELS bucket_c)
 
     add_test(
       NAME csr_test
@@ -934,11 +935,13 @@ if(BUILD_TESTS)
       NAME versionifier_test
       COMMAND ${PYTHON} ${CMAKE_SOURCE_DIR}/python/src/ccf/_versionifier.py
     )
+    set_property(TEST versionifier_test APPEND PROPERTY LABELS bucket_c)
 
     add_test(
       NAME github_version_lts_test
       COMMAND ${PYTHON} ${CMAKE_SOURCE_DIR}/tests/infra/github.py
     )
+    set_property(TEST github_version_lts_test APPEND PROPERTY LABELS bucket_c)
   endif()
 
   if(NOT TSAN)
@@ -983,6 +986,7 @@ if(BUILD_TESTS)
   add_e2e_test(
     NAME recovery_test
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/recovery.py
+    BUCKET bucket_b
     ADDITIONAL_ARGS
       ${ADDITIONAL_RECOVERY_ARGS}
       --constitution
@@ -1051,6 +1055,7 @@ if(BUILD_TESTS)
   add_e2e_test(
     NAME js_batched_stress_test
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_batched.py
+    BUCKET bucket_c
     ADDITIONAL_ARGS
       --js-app-bundle
       ${CMAKE_SOURCE_DIR}/src/apps/batched
@@ -1062,6 +1067,7 @@ if(BUILD_TESTS)
   add_e2e_test(
     NAME modules_test
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/js-modules/modules.py
+    BUCKET bucket_c
     ADDITIONAL_ARGS
       --package
       js_generic
@@ -1075,6 +1081,7 @@ if(BUILD_TESTS)
     NAME auth
     PYTHON_SCRIPT
       ${CMAKE_SOURCE_DIR}/tests/js-custom-authorization/custom_authorization.py
+    BUCKET bucket_c
     ADDITIONAL_ARGS
       --package
       js_generic
@@ -1100,6 +1107,7 @@ if(BUILD_TESTS)
     NAME governance_test
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/governance.py
     CONSTITUTION ${CONSTITUTION_ARGS}
+    BUCKET bucket_c
     ADDITIONAL_ARGS
       --initial-operator-count
       1
@@ -1110,6 +1118,7 @@ if(BUILD_TESTS)
   add_e2e_test(
     NAME code_update_test
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/code_update.py
+    BUCKET bucket_c
     ADDITIONAL_ARGS
       --js-app-bundle
       ${CMAKE_SOURCE_DIR}/samples/apps/logging/js
@@ -1136,6 +1145,7 @@ if(BUILD_TESTS)
   add_e2e_test(
     NAME e2e_logging
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_logging.py
+    BUCKET bucket_c
     ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/logging/js
   )
 
@@ -1158,25 +1168,29 @@ if(BUILD_TESTS)
     CONSTITUTION ${RBAC_CONSTITUTION_ARGS}
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/programmability.py
     LABEL snp
+    BUCKET bucket_c
   )
 
   # This test uses large requests (so too slow for SAN)
   if(NOT SAN)
     add_e2e_test(
       NAME e2e_limits
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/limits.py
+      BUCKET bucket_c
     )
   endif()
 
   add_e2e_test(
     NAME e2e_redirects
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/redirects.py
+    BUCKET bucket_c
     ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/logging/js
   )
 
   add_e2e_test(
     NAME e2e_logging_http2
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_logging.py
+    BUCKET bucket_c
     ADDITIONAL_ARGS
       --js-app-bundle
       ${CMAKE_SOURCE_DIR}/samples/apps/logging/js
@@ -1214,19 +1228,22 @@ if(BUILD_TESTS)
   add_e2e_test(
     NAME tls_stress_test
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/connections.py
+    BUCKET bucket_c
   )
 
   if(CLIENT_PROTOCOLS_TEST)
     add_e2e_test(
       NAME client_protocols
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/client_protocols.py
       LABEL protocolstest
+      BUCKET bucket_c
     )
   endif()
 
   add_e2e_test(
     NAME schema_test
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/schema.py
+    BUCKET bucket_b
     ADDITIONAL_ARGS
       --constitution
       ${CMAKE_SOURCE_DIR}/samples/constitutions/virtual/virtual_attestation_actions.js
@@ -1258,6 +1275,7 @@ if(BUILD_TESTS)
       NAME lts_compatibility
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/lts_compatibility.py
       LABEL e2e
+      BUCKET bucket_a
       ADDITIONAL_ARGS
         ${LTS_TEST_ARGS}
         --constitution
@@ -1283,6 +1301,7 @@ if(BUILD_TESTS)
   add_e2e_test(
     NAME nodes_test
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/nodes.py
+    BUCKET bucket_b
     ADDITIONAL_ARGS ${RECONFIG_TEST_ARGS} ${ROTATION_TEST_ARGS}
   )
 
@@ -1384,16 +1403,19 @@ if(BUILD_TESTS)
     add_e2e_test(
       NAME e2e_curl
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_curl.py
+      BUCKET bucket_c
     )
 
     add_e2e_test(
       NAME historical_query_cache_test
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/historical_query_cache.py
+      BUCKET bucket_c
     )
 
     add_e2e_test(
       NAME consistency_trace_validation
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/consistency_trace_validation.py
+      BUCKET bucket_c
     )
   endif()
 

cmake/common.cmake

@@ -90,13 +90,18 @@ function(add_test_bin name)
   add_san(${name})
 endfunction()
 
-# Helper for building end-to-end function tests using the python infrastructure
+# Helper for building end-to-end function tests using the python infrastructure.
+#
+# BUCKET assigns the test to one of the CI runner buckets (bucket_a, bucket_b,
+# bucket_c) so that .github/workflows/ci.yml can select the per-runner test set
+# with `ctest -L bucket_X`. Every PR-CI e2e test must be in exactly one bucket;
+# scripts/test-buckets-checks.sh flags unbucketed tests in `no_bucket:`.
 function(add_e2e_test)
   cmake_parse_arguments(
     PARSE_ARGV 0
     PARSED_ARGS
     ""
-    "NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT"
+    "NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT;BUCKET"
     "CONSTITUTION;ADDITIONAL_ARGS;CONFIGURATIONS"
   )
 
@@ -178,6 +183,14 @@ function(add_e2e_test)
       PROPERTY LABELS ${PARSED_ARGS_LABEL}
     )
 
+    if(PARSED_ARGS_BUCKET)
+      set_property(
+        TEST ${PARSED_ARGS_NAME}
+        APPEND
+        PROPERTY LABELS ${PARSED_ARGS_BUCKET}
+      )
+    endif()
+
     if(${PARSED_ARGS_CURL_CLIENT})
       set_property(
         TEST ${PARSED_ARGS_NAME}

cmake/gersemi_definitions.cmake

@@ -24,7 +24,7 @@ function(add_e2e_test)
     PARSE_ARGV 0
     PARSED_ARGS
     ""
-    "NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT"
+    "NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT;BUCKET"
     "CONSTITUTION;ADDITIONAL_ARGS;CONFIGURATIONS"
   )
 endfunction()

scripts/ci-checks.sh

@@ -48,6 +48,7 @@ CHECKS=(
   "Python format:python-format-checks.sh"
   "Python lint:python-lint-checks.sh"
   "Python types:python-types-checks.sh"
+  "CI test buckets:test-buckets-checks.sh"
 )
 
 declare -A PID_TO_IDX