From dd95717cc1cb741454ba43bbf9ecea377bb430c6 Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Wed, 30 Oct 2024 14:20:14 +0100
Subject: [PATCH 01/11] Remove ECJPAKE interoperability testing

We no longer have two (only partially distinct) implementations of ECJ-PAKE
cipher suites in TLS, now that the non-MBEDTLS_USE_PSA_CRYPTO implementation
is being removed.

We may want to add this testing back in the future, but we'll have to use an
old Mbed TLS instead of a differently-built one.
https://github.com/Mbed-TLS/mbedtls/issues/9740

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 tests/scripts/components-configuration-tls.sh | 33 -------------------
 1 file changed, 33 deletions(-)

diff --git a/tests/scripts/components-configuration-tls.sh b/tests/scripts/components-configuration-tls.sh
index b8834d60951e..e01a5989df67 100644
--- a/tests/scripts/components-configuration-tls.sh
+++ b/tests/scripts/components-configuration-tls.sh
@@ -184,39 +184,6 @@ component_test_config_thread () {
     tests/ssl-opt.sh -f 'ECJPAKE.*nolog'
 }
 
-# We're not aware of any other (open source) implementation of EC J-PAKE in TLS
-# that we could use for interop testing. However, we now have sort of two
-# implementations ourselves: one using PSA, the other not. At least test that
-# these two interoperate with each other.
-component_test_tls1_2_ecjpake_compatibility () {
-    msg "build: TLS1.2 server+client w/ EC-JPAKE w/o USE_PSA"
-    scripts/config.py set MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED
-    # Explicitly make lib first to avoid a race condition:
-    # https://github.com/Mbed-TLS/mbedtls/issues/8229
-    make lib
-    make -C programs ssl/ssl_server2 ssl/ssl_client2
-    cp programs/ssl/ssl_server2 s2_no_use_psa
-    cp programs/ssl/ssl_client2 c2_no_use_psa
-
-    msg "build: TLS1.2 server+client w/ EC-JPAKE w/ USE_PSA"
-    scripts/config.py set MBEDTLS_USE_PSA_CRYPTO
-    make clean
-    make lib
-    make -C programs ssl/ssl_server2 ssl/ssl_client2
-    make -C programs test/udp_proxy test/query_compile_time_config
-
-    msg "test: server w/o USE_PSA - client w/ USE_PSA, text password"
-    P_SRV=../s2_no_use_psa tests/ssl-opt.sh -f "ECJPAKE: working, TLS"
-    msg "test: server w/o USE_PSA - client w/ USE_PSA, opaque password"
-    P_SRV=../s2_no_use_psa tests/ssl-opt.sh -f "ECJPAKE: opaque password client only, working, TLS"
-    msg "test: client w/o USE_PSA - server w/ USE_PSA, text password"
-    P_CLI=../c2_no_use_psa tests/ssl-opt.sh -f "ECJPAKE: working, TLS"
-    msg "test: client w/o USE_PSA - server w/ USE_PSA, opaque password"
-    P_CLI=../c2_no_use_psa tests/ssl-opt.sh -f "ECJPAKE: opaque password server only, working, TLS"
-
-    rm s2_no_use_psa c2_no_use_psa
-}
-
 component_test_tls1_2_ccm_psk () {
     msg "build: configs/config-ccm-psk-tls1_2.h"
     cp configs/config-ccm-psk-tls1_2.h "$CONFIG_H"

From 45fdbdd14bdaf0b55c7367f50149c6b2fee0991a Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Tue, 18 Jun 2024 20:32:08 +0200
Subject: [PATCH 02/11] Record build steps in the outcome file

Add an outcome line when all.sh calls make.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 tests/scripts/quiet/make     | 56 ++++++++++++++++++++++++++++++++++++
 tests/scripts/quiet/quiet.sh |  5 ++--
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/tests/scripts/quiet/make b/tests/scripts/quiet/make
index 920e5b875fd1..a76315c600a8 100755
--- a/tests/scripts/quiet/make
+++ b/tests/scripts/quiet/make
@@ -17,3 +17,59 @@ NO_SILENCE=" --version | test "
 TOOL="make"
 
 . "$(dirname "$0")/quiet.sh"
+EXIT_STATUS=$?
+
+log_outcome () {
+    targets=
+    skip=
+    for arg in "$@"; do
+        if [ -n "$skip" ]; then
+            skip=
+            continue
+        fi
+        case $arg in
+            --assume-new|--assume-old|--directory|--file| \
+            --include-dir|--makefile|--new-file|--old-file|--what-if| \
+            -C|-I|-W|-f|-k|-o) # Option with separate argument
+                skip=1; continue;;
+            -*) continue;; # Option
+            *=*) continue;; # Variable assignment
+            *[!-+./0-9@A-Z_a-z]*) # Target with problematic character
+                targets="$targets ${arg%%[!-+./0-9@A-Z_a-z]*}...";;
+            *) # Normal target
+                targets="$targets $arg";;
+        esac
+    done
+    if [ -n "$targets" ]; then
+        targets=${targets# }
+    else
+        targets=all
+    fi
+
+    # We have a single pass/fail status. This is not accurate when there are
+    # multiple targets: it's possible that some passed and some failed.
+    # To figure out which targets passed when the overall result is a failure,
+    # we'd have to do some complex parsing of logs.
+    if [ $EXIT_STATUS -eq 0 ]; then
+        result=PASS
+    else
+        result=FAIL
+    fi
+    cause= # Identifying failure causes would be nice, but difficult
+
+    for target in $targets; do
+        if [ "$target" = "clean" ]; then
+            # Boring
+            continue
+        fi
+        echo >>"${MBEDTLS_TEST_OUTCOME_FILE}" \
+             "${MBEDTLS_TEST_PLATFORM};${MBEDTLS_TEST_CONFIGURATION};${TOOL};${target};${result};${cause}"
+    done
+}
+
+if [ -n "${MBEDTLS_TEST_OUTCOME_FILE}" ] &&
+   [ -n "${MBEDTLS_TEST_CONFIGURATION}" ]; then
+    log_outcome "$@"
+fi
+
+exit $EXIT_STATUS
diff --git a/tests/scripts/quiet/quiet.sh b/tests/scripts/quiet/quiet.sh
index 0f26184d0d77..e81243d2bf40 100644
--- a/tests/scripts/quiet/quiet.sh
+++ b/tests/scripts/quiet/quiet.sh
@@ -59,7 +59,7 @@ fi
 
 if [[ " $@ " =~ $NO_SILENCE || -n "${VERBOSE_LOGS}" ]]; then
     # Run original command with no output supression
-    exec "${ORIGINAL_TOOL}" "$@"
+    "${ORIGINAL_TOOL}" "$@"
 else
     # Run original command and capture output & exit status
     TMPFILE=$(mktemp "quiet-${TOOL}.XXXXXX")
@@ -75,5 +75,6 @@ else
     rm "${TMPFILE}"
 
     # Propagate the exit status
-    exit $EXIT_STATUS
+    set_status () { return $1; }
+    set_status $EXIT_STATUS
 fi

From 5dfa99ff9395f9d4dac166fb85058741a20eb0b5 Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Wed, 19 Jun 2024 20:02:14 +0200
Subject: [PATCH 03/11] Set MBEDTLS_TEST_CONFIGURATION to distinct values when
 doing multiple builds

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 tests/scripts/components-build-system.sh      |  6 ++++
 tests/scripts/components-compiler.sh          |  5 +++
 .../components-configuration-crypto.sh        | 13 ++++++++
 tests/scripts/components-configuration.sh     |  4 +++
 tests/scripts/components-platform.sh          | 33 +++++++++++++++++++
 5 files changed, 61 insertions(+)

diff --git a/tests/scripts/components-build-system.sh b/tests/scripts/components-build-system.sh
index 3047e7625201..dccb57dc4ca9 100644
--- a/tests/scripts/components-build-system.sh
+++ b/tests/scripts/components-build-system.sh
@@ -193,15 +193,18 @@ component_build_cmake_custom_config_file () {
     cd "$OUT_OF_SOURCE_DIR"
 
     # Build once to get the generated files (which need an intact config file)
+    MBEDTLS_TEST_CONFIGURATION="$current_component/out_of_tree/default"
     cmake "$MBEDTLS_ROOT_DIR"
     make
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/out_of_tree/MBEDTLS_CONFIG_FILE"
     msg "build: cmake with -DMBEDTLS_CONFIG_FILE"
     scripts/config.py -w full_config.h full
     echo '#error "cmake -DMBEDTLS_CONFIG_FILE is not working."' > "$MBEDTLS_ROOT_DIR/$CONFIG_H"
     cmake -DGEN_FILES=OFF -DMBEDTLS_CONFIG_FILE=full_config.h "$MBEDTLS_ROOT_DIR"
     make
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/out_of_tree/MBEDTLS_USER_CONFIG_FILE"
     msg "build: cmake with -DMBEDTLS_CONFIG_FILE + -DMBEDTLS_USER_CONFIG_FILE"
     # In the user config, disable one feature (for simplicity, pick a feature
     # that nothing else depends on).
@@ -217,6 +220,7 @@ component_build_cmake_custom_config_file () {
     rm -rf "$OUT_OF_SOURCE_DIR"
 
     # Now repeat the test for an in-tree build:
+    MBEDTLS_TEST_CONFIGURATION="$current_component/in_tree/default"
 
     # Restore config for the in-tree test
     mv include/mbedtls_config_in_tree_copy.h "$CONFIG_H"
@@ -225,12 +229,14 @@ component_build_cmake_custom_config_file () {
     cmake .
     make
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/in_tree/MBEDTLS_CONFIG_FILE"
     msg "build: cmake (in-tree) with -DMBEDTLS_CONFIG_FILE"
     scripts/config.py -w full_config.h full
     echo '#error "cmake -DMBEDTLS_CONFIG_FILE is not working."' > "$MBEDTLS_ROOT_DIR/$CONFIG_H"
     cmake -DGEN_FILES=OFF -DMBEDTLS_CONFIG_FILE=full_config.h .
     make
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/in_tree/MBEDTLS_USER_CONFIG_FILE"
     msg "build: cmake (in-tree) with -DMBEDTLS_CONFIG_FILE + -DMBEDTLS_USER_CONFIG_FILE"
     # In the user config, disable one feature (for simplicity, pick a feature
     # that nothing else depends on).
diff --git a/tests/scripts/components-compiler.sh b/tests/scripts/components-compiler.sh
index 5d22735caf74..ea52390f1be8 100644
--- a/tests/scripts/components-compiler.sh
+++ b/tests/scripts/components-compiler.sh
@@ -25,6 +25,7 @@ test_build_opt () {
     info=$1 cc=$2; shift 2
     $cc --version
     for opt in "$@"; do
+          MBEDTLS_TEST_CONFIGURATION="$current_component$opt"
           msg "build/test: $cc $opt, $info" # ~ 30s
           make CC="$cc" CFLAGS="$opt -std=c99 -pedantic -Wall -Wextra -Werror"
           # We're confident enough in compilers to not run _all_ the tests,
@@ -82,6 +83,7 @@ support_test_gcc_earliest_opt () {
 }
 
 component_build_mingw () {
+    MBEDTLS_TEST_CONFIGURATION="$current_component/w64-mingw32/default/static"
     msg "build: Windows cross build - mingw64, make (Link Library)" # ~ 30s
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -Wall -Wextra -maes -msse2 -mpclmul' WINDOWS_BUILD=1 lib programs
 
@@ -89,11 +91,13 @@ component_build_mingw () {
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -maes -msse2 -mpclmul' WINDOWS_BUILD=1 tests
     make WINDOWS_BUILD=1 clean
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/w64-mingw32/default/shared"
     msg "build: Windows cross build - mingw64, make (DLL)" # ~ 30s
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -Wall -Wextra -maes -msse2 -mpclmul' WINDOWS_BUILD=1 SHARED=1 lib programs
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -Wall -Wextra -maes -msse2 -mpclmul' WINDOWS_BUILD=1 SHARED=1 tests
     make WINDOWS_BUILD=1 clean
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/w64-mingw32/no_AESNI/static"
     msg "build: Windows cross build - mingw64, make (Library only, default config without MBEDTLS_AESNI_C)" # ~ 30s
     ./scripts/config.py unset MBEDTLS_AESNI_C #
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 lib
@@ -133,6 +137,7 @@ component_test_zeroize () {
 
     for optimization_flag in -O2 -O3 -Ofast -Os; do
         for compiler in clang gcc; do
+            MBEDTLS_TEST_CONFIGURATION="$current_component/$optimization_flag/$compiler"
             msg "test: $compiler $optimization_flag, mbedtls_platform_zeroize()"
             make programs CC="$compiler" DEBUG=1 CFLAGS="$optimization_flag"
             gdb -ex "$gdb_disable_aslr" -x tests/scripts/test_zeroize.gdb -nw -batch -nx 2>&1 | tee test_zeroize.log
diff --git a/tests/scripts/components-configuration-crypto.sh b/tests/scripts/components-configuration-crypto.sh
index 6ee0f919d700..783012250db3 100644
--- a/tests/scripts/components-configuration-crypto.sh
+++ b/tests/scripts/components-configuration-crypto.sh
@@ -2283,6 +2283,7 @@ component_build_aes_variations () {
     # Test that all the combinations build cleanly.
 
     MBEDTLS_ROOT_DIR="$PWD"
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both_directions"
     msg "build: aes.o for all combinations of relevant config options"
 
     build_test_config_combos ${BUILTIN_SRC_PATH}/aes.o validate_aes_config_variations \
@@ -2291,6 +2292,8 @@ component_build_aes_variations () {
         "MBEDTLS_AESNI_C" "MBEDTLS_AESCE_C" "MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH"
 
     cd "$MBEDTLS_ROOT_DIR"
+
+    MBEDTLS_TEST_CONFIGURATION="$current_component/no_decrypt"
     msg "build: aes.o for all combinations of relevant config options + BLOCK_CIPHER_NO_DECRYPT"
 
     # MBEDTLS_BLOCK_CIPHER_NO_DECRYPT is incompatible with ECB in PSA, CBC/XTS/NIST_KW/DES,
@@ -2336,11 +2339,13 @@ END
         #define PSA_WANT_ALG_SHA3_512  1
 END
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/unrolled"
     msg "all loops unrolled"
     make clean
     make -C tests ../tf-psa-crypto/tests/test_suite_shax CFLAGS="-DMBEDTLS_SHA3_THETA_UNROLL=1 -DMBEDTLS_SHA3_PI_UNROLL=1 -DMBEDTLS_SHA3_CHI_UNROLL=1 -DMBEDTLS_SHA3_RHO_UNROLL=1"
     ./tf-psa-crypto/tests/test_suite_shax
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/rolled"
     msg "all loops rolled up"
     make clean
     make -C tests ../tf-psa-crypto/tests/test_suite_shax CFLAGS="-DMBEDTLS_SHA3_THETA_UNROLL=0 -DMBEDTLS_SHA3_PI_UNROLL=0 -DMBEDTLS_SHA3_CHI_UNROLL=0 -DMBEDTLS_SHA3_RHO_UNROLL=0"
@@ -2368,11 +2373,13 @@ component_build_aes_aesce_armcc () {
     scripts/config.py unset MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT
     scripts/config.py set MBEDTLS_HAVE_ASM
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both"
     msg "AESCE, build with default configuration."
     scripts/config.py set MBEDTLS_AESCE_C
     scripts/config.py unset MBEDTLS_AES_USE_HARDWARE_ONLY
     helper_armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8-a+crypto"
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware"
     msg "AESCE, build AESCE only"
     scripts/config.py set MBEDTLS_AESCE_C
     scripts/config.py set MBEDTLS_AES_USE_HARDWARE_ONLY
@@ -2536,20 +2543,24 @@ component_test_block_cipher_no_decrypt_aesni () {
     config_block_cipher_no_decrypt
 
     # test AESNI intrinsics
+    MBEDTLS_TEST_CONFIGURATION="$current_component/AESNI/intrinsics"
     helper_block_cipher_no_decrypt_build_test \
         -s "MBEDTLS_AESNI_C" \
         -c "-mpclmul -msse2 -maes"
 
     # test AESNI assembly
+    MBEDTLS_TEST_CONFIGURATION="$current_component/AESNI/assembly"
     helper_block_cipher_no_decrypt_build_test \
         -s "MBEDTLS_AESNI_C" \
         -c "-mno-pclmul -mno-sse2 -mno-aes"
 
     # test AES C implementation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/software"
     helper_block_cipher_no_decrypt_build_test \
         -u "MBEDTLS_AESNI_C"
 
     # test AESNI intrinsics for i386 target
+    MBEDTLS_TEST_CONFIGURATION="$current_component/AESNI/intrinsics/m32"
     helper_block_cipher_no_decrypt_build_test \
         -s "MBEDTLS_AESNI_C" \
         -c "-m32 -mpclmul -msse2 -maes" \
@@ -2671,6 +2682,7 @@ component_test_psa_crypto_drivers () {
 }
 
 component_build_psa_config_file () {
+    MBEDTLS_TEST_CONFIGURATION="$current_component/MBEDTLS_PSA_CRYPTO_CONFIG_FILE"
     msg "build: make with MBEDTLS_PSA_CRYPTO_CONFIG_FILE" # ~40s
     scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG
     cp "$CRYPTO_CONFIG_H" psa_test_config.h
@@ -2680,6 +2692,7 @@ component_build_psa_config_file () {
     programs/test/query_compile_time_config MBEDTLS_CMAC_C
     make clean
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/MBEDTLS_PSA_CRYPTO_USER_CONFIG_FILE"
     msg "build: make with MBEDTLS_PSA_CRYPTO_CONFIG_FILE + MBEDTLS_PSA_CRYPTO_USER_CONFIG_FILE" # ~40s
     # In the user config, disable one feature and its dependencies, which will
     # reflect on the mbedtls configuration so we can query it with
diff --git a/tests/scripts/components-configuration.sh b/tests/scripts/components-configuration.sh
index 683ac848874a..c4cdbd95c952 100644
--- a/tests/scripts/components-configuration.sh
+++ b/tests/scripts/components-configuration.sh
@@ -237,9 +237,11 @@ component_build_tfm () {
     # configs/config-tfm.h, tested via test-ref-configs.pl.
     cp configs/config-tfm.h "$CONFIG_H"
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/thumb2/clang"
     msg "build: TF-M config, clang, armv7-m thumb2"
     make lib CC="clang" CFLAGS="--target=arm-linux-gnueabihf -march=armv7-m -mthumb -Os -std=c99 -Werror -Wall -Wextra -Wwrite-strings -Wpointer-arith -Wimplicit-fallthrough -Wshadow -Wvla -Wformat=2 -Wno-format-nonliteral -Wshadow -Wasm-operand-widths -Wunused -I../tests/include/spe"
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/native/gcc"
     msg "build: TF-M config, gcc native build"
     make clean
     make lib CC="gcc" CFLAGS="-Os -std=c99 -Werror -Wall -Wextra -Wwrite-strings -Wpointer-arith -Wshadow -Wvla -Wformat=2 -Wno-format-nonliteral -Wshadow -Wformat-signedness -Wlogical-op -I../tests/include/spe"
@@ -287,6 +289,7 @@ component_test_no_platform () {
 }
 
 component_build_mbedtls_config_file () {
+    MBEDTLS_TEST_CONFIGURATION="$current_component/MBEDTLS_PSA_CRYPTO_CONFIG_FILE"
     msg "build: make with MBEDTLS_CONFIG_FILE" # ~40s
     scripts/config.py -w full_config.h full
     echo '#error "MBEDTLS_CONFIG_FILE is not working"' >"$CONFIG_H"
@@ -295,6 +298,7 @@ component_build_mbedtls_config_file () {
     programs/test/query_compile_time_config MBEDTLS_NIST_KW_C
     make clean
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/MBEDTLS_USER_CONFIG_FILE"
     msg "build: make with MBEDTLS_CONFIG_FILE + MBEDTLS_USER_CONFIG_FILE"
     # In the user config, disable one feature (for simplicity, pick a feature
     # that nothing else depends on).
diff --git a/tests/scripts/components-platform.sh b/tests/scripts/components-platform.sh
index a8c8c7befc88..03ad7d7e44dc 100644
--- a/tests/scripts/components-platform.sh
+++ b/tests/scripts/components-platform.sh
@@ -111,12 +111,14 @@ component_test_aesni () { # ~ 60s
     # AESNI detection will fallback to the plain C implementation, so the tests will instead
     # exercise the plain C impl).
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/default"
     msg "build: default config with different AES implementations"
     scripts/config.py set MBEDTLS_AESNI_C
     scripts/config.py unset MBEDTLS_AES_USE_HARDWARE_ONLY
     scripts/config.py set MBEDTLS_HAVE_ASM
 
     # test the intrinsics implementation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/intrinsics"
     msg "AES tests, test intrinsics"
     make clean
     make CC=gcc CFLAGS='-Werror -Wall -Wextra -mpclmul -msse2 -maes'
@@ -124,6 +126,7 @@ component_test_aesni () { # ~ 60s
     ./programs/test/selftest aes | grep "AESNI code" | grep -q "intrinsics"
 
     # test the asm implementation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/assembly"
     msg "AES tests, test assembly"
     make clean
     make CC=gcc CFLAGS='-Werror -Wall -Wextra -mno-pclmul -mno-sse2 -mno-aes'
@@ -131,6 +134,7 @@ component_test_aesni () { # ~ 60s
     ./programs/test/selftest aes | grep "AESNI code" | grep -q "assembly"
 
     # test the plain C implementation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/software"
     scripts/config.py unset MBEDTLS_AESNI_C
     scripts/config.py unset MBEDTLS_AES_USE_HARDWARE_ONLY
     msg "AES tests, plain C"
@@ -142,6 +146,7 @@ component_test_aesni () { # ~ 60s
     grep -q "AES note: built-in implementation." ./programs/test/selftest
 
     # test the intrinsics implementation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/intrinsics"
     scripts/config.py set MBEDTLS_AESNI_C
     scripts/config.py set MBEDTLS_AES_USE_HARDWARE_ONLY
     msg "AES tests, test AESNI only"
@@ -168,6 +173,7 @@ component_test_aesni_m32 () { # ~ 60s
     scripts/config.py set MBEDTLS_HAVE_ASM
 
     # test the intrinsics implementation with gcc
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/intrinsics"
     msg "AES tests, test intrinsics (gcc)"
     make clean
     make CC=gcc CFLAGS='-m32 -Werror -Wall -Wextra' LDFLAGS='-m32'
@@ -177,6 +183,7 @@ component_test_aesni_m32 () { # ~ 60s
     grep -q "AES note: built-in implementation." ./programs/test/selftest
     grep -q mbedtls_aesni_has_support ./programs/test/selftest
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/intrinsics"
     scripts/config.py set MBEDTLS_AESNI_C
     scripts/config.py set MBEDTLS_AES_USE_HARDWARE_ONLY
     msg "AES tests, test AESNI only"
@@ -221,16 +228,19 @@ component_build_aes_armce () {
     scripts/config.py set MBEDTLS_AESCE_C
     scripts/config.py set MBEDTLS_AES_USE_HARDWARE_ONLY
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/aarch64"
     msg "MBEDTLS_AES_USE_HARDWARE_ONLY, clang, aarch64"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=aarch64-linux-gnu -march=armv8-a+crypto"
     msg "clang, test aarch64 crypto instructions built"
     grep -E 'aes[a-z]+\s*[qv]' ${BUILTIN_SRC_PATH}/aesce.s
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/arm"
     msg "MBEDTLS_AES_USE_HARDWARE_ONLY, clang, arm"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a72+crypto -marm"
     msg "clang, test A32 crypto instructions built"
     grep -E 'aes[0-9a-z]+.[0-9]\s*[qv]' ${BUILTIN_SRC_PATH}/aesce.s
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/thumb"
     msg "MBEDTLS_AES_USE_HARDWARE_ONLY, clang, thumb"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a32+crypto -mthumb"
     msg "clang, test T32 crypto instructions built"
@@ -238,16 +248,19 @@ component_build_aes_armce () {
 
     scripts/config.py unset MBEDTLS_AES_USE_HARDWARE_ONLY
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/aarch64"
     msg "MBEDTLS_AES_USE_both, clang, aarch64"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=aarch64-linux-gnu -march=armv8-a+crypto"
     msg "clang, test aarch64 crypto instructions built"
     grep -E 'aes[a-z]+\s*[qv]' ${BUILTIN_SRC_PATH}/aesce.s
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/arm"
     msg "MBEDTLS_AES_USE_both, clang, arm"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a72+crypto -marm"
     msg "clang, test A32 crypto instructions built"
     grep -E 'aes[0-9a-z]+.[0-9]\s*[qv]' ${BUILTIN_SRC_PATH}/aesce.s
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/thumb"
     msg "MBEDTLS_AES_USE_both, clang, thumb"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a32+crypto -mthumb"
     msg "clang, test T32 crypto instructions built"
@@ -255,16 +268,19 @@ component_build_aes_armce () {
 
     scripts/config.py unset MBEDTLS_AESCE_C
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/software_only/aarch64"
     msg "no MBEDTLS_AESCE_C, clang, aarch64"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=aarch64-linux-gnu -march=armv8-a"
     msg "clang, test aarch64 crypto instructions not built"
     not grep -E 'aes[a-z]+\s*[qv]' ${BUILTIN_SRC_PATH}/aesce.s
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/software_only/arm"
     msg "no MBEDTLS_AESCE_C, clang, arm"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a72 -marm"
     msg "clang, test A32 crypto instructions not built"
     not grep -E 'aes[0-9a-z]+.[0-9]\s*[qv]' ${BUILTIN_SRC_PATH}/aesce.s
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/software_only/thumb"
     msg "no MBEDTLS_AESCE_C, clang, thumb"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a32 -mthumb"
     msg "clang, test T32 crypto instructions not built"
@@ -281,11 +297,13 @@ component_build_sha_armce () {
 
     # Test variations of SHA256 Armv8 crypto extensions
     scripts/config.py set MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY
+        MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/aarch64"
         msg "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY clang, aarch64"
         make -B library/../${BUILTIN_SRC_PATH}/sha256.o library/../${BUILTIN_SRC_PATH}/sha256.s CC=clang CFLAGS="--target=aarch64-linux-gnu -march=armv8-a+crypto"
         msg "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY clang, test aarch64 crypto instructions built"
         grep -E 'sha256[a-z0-9]+\s+[qv]' ${BUILTIN_SRC_PATH}/sha256.s
 
+        MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/arm"
         msg "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY clang, arm"
         make -B library/../${BUILTIN_SRC_PATH}/sha256.o library/../${BUILTIN_SRC_PATH}/sha256.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a72+crypto -marm"
         msg "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY clang, test A32 crypto instructions built"
@@ -295,6 +313,7 @@ component_build_sha_armce () {
 
     # test the deprecated form of the config option
     scripts/config.py set MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY
+        MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/thumb"
         msg "MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY clang, thumb"
         make -B library/../${BUILTIN_SRC_PATH}/sha256.o library/../${BUILTIN_SRC_PATH}/sha256.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a32+crypto -mthumb"
         msg "MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY clang, test T32 crypto instructions built"
@@ -302,6 +321,7 @@ component_build_sha_armce () {
     scripts/config.py unset MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY
 
     scripts/config.py set MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT
+        MBEDTLS_TEST_CONFIGURATION="$current_component/both/aarch64"
         msg "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT clang, aarch64"
         make -B library/../${BUILTIN_SRC_PATH}/sha256.o library/../${BUILTIN_SRC_PATH}/sha256.s CC=clang CFLAGS="--target=aarch64-linux-gnu -march=armv8-a+crypto"
         msg "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT clang, test aarch64 crypto instructions built"
@@ -311,9 +331,11 @@ component_build_sha_armce () {
 
     # test the deprecated form of the config option
     scripts/config.py set MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT
+        MBEDTLS_TEST_CONFIGURATION="$current_component/both/arm"
         msg "MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT clang, arm"
         make -B library/../${BUILTIN_SRC_PATH}/sha256.o library/../${BUILTIN_SRC_PATH}/sha256.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a72+crypto -marm -std=c99"
 
+        MBEDTLS_TEST_CONFIGURATION="$current_component/both/thumb"
         msg "MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT clang, thumb"
         make -B library/../${BUILTIN_SRC_PATH}/sha256.o library/../${BUILTIN_SRC_PATH}/sha256.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a32+crypto -mthumb"
         msg "MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT clang, test T32 crypto instructions built"
@@ -533,15 +555,18 @@ component_build_arm_clang_thumb () {
 
     scripts/config.py baremetal
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/thumb2/-Os"
     msg "build: clang thumb 2, make"
     make clean
     make CC="clang" CFLAGS='-std=c99 -Werror -Os --target=arm-linux-gnueabihf -march=armv7-m -mthumb' lib
 
     # Some Thumb 1 asm is sensitive to optimisation level, so test both -O0 and -Os
+    MBEDTLS_TEST_CONFIGURATION="$current_component/thumb/-O0"
     msg "build: clang thumb 1 -O0, make"
     make clean
     make CC="clang" CFLAGS='-std=c99 -Werror -O0 --target=arm-linux-gnueabihf -mcpu=arm1136j-s -mthumb' lib
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/thumb/-Os"
     msg "build: clang thumb 1 -Os, make"
     make clean
     make CC="clang" CFLAGS='-std=c99 -Werror -Os --target=arm-linux-gnueabihf -mcpu=arm1136j-s -mthumb' lib
@@ -576,24 +601,31 @@ component_build_armcc () {
     # Compile mostly with -O1 since some Arm inline assembly is disabled for -O0.
 
     # ARM Compiler 6 - Target ARMv7-A
+    MBEDTLS_TEST_CONFIGURATION="$current_component/v7a/-O1"
     helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv7-a"
 
     # ARM Compiler 6 - Target ARMv7-M
+    MBEDTLS_TEST_CONFIGURATION="$current_component/v7m/-O1"
     helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv7-m"
 
     # ARM Compiler 6 - Target ARMv7-M+DSP
+    MBEDTLS_TEST_CONFIGURATION="$current_component/v7m+dsp/-O1"
     helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv7-m+dsp"
 
     # ARM Compiler 6 - Target ARMv8-A - AArch32
+    MBEDTLS_TEST_CONFIGURATION="$current_component/v82a/-O1"
     helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv8.2-a"
 
     # ARM Compiler 6 - Target ARMv8-M
+    MBEDTLS_TEST_CONFIGURATION="$current_component/v8m/-O1"
     helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv8-m.main"
 
     # ARM Compiler 6 - Target Cortex-M0 - no optimisation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/m0/-O0"
     helper_armc6_build_test "-O0 --target=arm-arm-none-eabi -mcpu=cortex-m0"
 
     # ARM Compiler 6 - Target Cortex-M0
+    MBEDTLS_TEST_CONFIGURATION="$current_component/m0/-Os"
     helper_armc6_build_test "-Os --target=arm-arm-none-eabi -mcpu=cortex-m0"
 
     # ARM Compiler 6 - Target ARMv8.2-A - AArch64
@@ -601,6 +633,7 @@ component_build_armcc () {
     # Re-enable MBEDTLS_AESCE_C as this should be supported by the version of armclang
     # that we have in our CI
     scripts/config.py set MBEDTLS_AESCE_C
+    MBEDTLS_TEST_CONFIGURATION="$current_component/v82a+crypto/-O1"
     helper_armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8.2-a+crypto"
 }
 

From 6d71fd160de9ab6cb46e88535a4ad534c35852ab Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Thu, 20 Jun 2024 14:42:50 +0200
Subject: [PATCH 04/11] Ignore weirdness from cmake

Some versions on CMake (on some platforms?) run make (not $(MAKE)) under the
hood on targets named cmTC_* (CMake TryCompile). Ignore those calls.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 tests/scripts/quiet/make | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/scripts/quiet/make b/tests/scripts/quiet/make
index a76315c600a8..bcb40bfd0477 100755
--- a/tests/scripts/quiet/make
+++ b/tests/scripts/quiet/make
@@ -58,10 +58,10 @@ log_outcome () {
     cause= # Identifying failure causes would be nice, but difficult
 
     for target in $targets; do
-        if [ "$target" = "clean" ]; then
-            # Boring
-            continue
-        fi
+        case "$target" in
+            "clean"|"neat") continue;; # Boring
+            cmTC_*) continue;; # Weirdness from CMake
+        esac
         echo >>"${MBEDTLS_TEST_OUTCOME_FILE}" \
              "${MBEDTLS_TEST_PLATFORM};${MBEDTLS_TEST_CONFIGURATION};${TOOL};${target};${result};${cause}"
     done

From ad99f85e6e64895b3d1bebc6bbc0adce4db66518 Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Wed, 30 Oct 2024 15:44:08 +0100
Subject: [PATCH 05/11] Write an outcome file line for every component

This should make failures.csv on the CI be a complete list of failures
(although it will still have incomplete information on what has failed, if
what failed wasn't tracked individually).

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 tests/scripts/all-core.sh | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/tests/scripts/all-core.sh b/tests/scripts/all-core.sh
index 3b5a053073d0..dcca8116d066 100644
--- a/tests/scripts/all-core.sh
+++ b/tests/scripts/all-core.sh
@@ -790,6 +790,19 @@ pre_prepare_outcome_file () {
     fi
 }
 
+## write_outcome_line SUITE CASE RESULT [CAUSE]
+## Write a line in the outcome file if enabled.
+## Report $MBEDTLS_TEST_PLATFORM, $MBEDTLS_TEST_CONFIGURATION and the
+## supplied arguments.
+write_outcome_line () {
+    if [ -z "$MBEDTLS_TEST_OUTCOME_FILE" ]; then
+        return
+    fi
+    printf '%s;%s;%s;%s;%s;%s\n' >>"$MBEDTLS_TEST_OUTCOME_FILE" \
+           "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \
+           "$1" "$2" "$3" "${4-}"
+}
+
 pre_print_configuration () {
     if [ $QUIET -eq 1 ]; then
         return
@@ -983,6 +996,12 @@ run_component () {
         fi
     fi
 
+    if [ $component_status -eq 0 ]; then
+        write_outcome_line "all.sh" "whole" "PASS"
+    else
+        write_outcome_line "all.sh" "whole" "FAIL" "$component_status"
+    fi
+
     # Restore the build tree to a clean state.
     cleanup
     unset current_component

From ee358ce2ed94bed07b611779c29832706c0e29fa Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Wed, 30 Oct 2024 15:50:27 +0100
Subject: [PATCH 06/11] Update framework

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 framework | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/framework b/framework
index d68446c9da02..60063336977a 160000
--- a/framework
+++ b/framework
@@ -1 +1 @@
-Subproject commit d68446c9da02e536279a7aaa5a3c9850742ba30c
+Subproject commit 60063336977a591958d3f5414d89c492ba76220a

From 8a782ea07e0080b199da4a6d9416ea834025a830 Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Wed, 30 Oct 2024 17:58:11 +0100
Subject: [PATCH 07/11] Align test_tfm_config_no_p256m with its reference

component_test_tfm_config_no_p256m was not building sample programs, but
component_test_tfm_config_p256m_driver_accel_ec was. This is now flagged as
a discrepancy by outcome analysis.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 tests/scripts/components-configuration-crypto.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/scripts/components-configuration-crypto.sh b/tests/scripts/components-configuration-crypto.sh
index 783012250db3..9c95ebece81c 100644
--- a/tests/scripts/components-configuration-crypto.sh
+++ b/tests/scripts/components-configuration-crypto.sh
@@ -1448,7 +1448,6 @@ component_test_tfm_config_p256m_driver_accel_ec () {
 
     common_tfm_config
 
-    # Build crypto library
     make CC=$ASAN_CC CFLAGS="$ASAN_CFLAGS -I../tests/include/spe" LDFLAGS="$ASAN_CFLAGS"
 
     # Make sure any built-in EC alg was not re-enabled by accident (additive config)
@@ -1483,7 +1482,7 @@ component_test_tfm_config_no_p256m () {
     echo "#undef MBEDTLS_PSA_P256M_DRIVER_ENABLED" >> "$CONFIG_H"
 
     msg "build: TF-M config without p256m"
-    make CFLAGS='-Werror -Wall -Wextra -I../tests/include/spe' tests
+    make CFLAGS='-Werror -Wall -Wextra -I../tests/include/spe'
 
     # Check that p256m was not built
     not grep p256_ecdsa_ library/libmbedcrypto.a

From f8f192516e2e3078e5158def81ef3c25182321f4 Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Wed, 30 Oct 2024 18:57:17 +0100
Subject: [PATCH 08/11] Don't log `make -q` calls in the outcome file

This comes up in `psa_collect_statuses.py` and possibly (I didn't finish the
analysis) some CMake builds, which were reporting duplicate `make` steps.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 tests/scripts/quiet/make | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/scripts/quiet/make b/tests/scripts/quiet/make
index bcb40bfd0477..669a3aa2eb66 100755
--- a/tests/scripts/quiet/make
+++ b/tests/scripts/quiet/make
@@ -28,6 +28,8 @@ log_outcome () {
             continue
         fi
         case $arg in
+            --question|-q) # do-nothing option
+                return;;
             --assume-new|--assume-old|--directory|--file| \
             --include-dir|--makefile|--new-file|--old-file|--what-if| \
             -C|-I|-W|-f|-k|-o) # Option with separate argument

From b3940ab4b4df1e96a488e580d0b0e9e709a02f19 Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Wed, 30 Oct 2024 18:59:02 +0100
Subject: [PATCH 09/11] Log `make -q` command lines the outcome file

As a debugging help, log the `make` command line in the outcome file. This
is not accurate if a command line argument contains some special
characters, but good enough to help debugging. Hopefully this will only be
temporary.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 tests/scripts/quiet/make | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/tests/scripts/quiet/make b/tests/scripts/quiet/make
index 669a3aa2eb66..b31389122ff2 100755
--- a/tests/scripts/quiet/make
+++ b/tests/scripts/quiet/make
@@ -19,6 +19,9 @@ TOOL="make"
 . "$(dirname "$0")/quiet.sh"
 EXIT_STATUS=$?
 
+nl='
+' # set the variable to a newline
+
 log_outcome () {
     targets=
     skip=
@@ -57,7 +60,19 @@ log_outcome () {
     else
         result=FAIL
     fi
-    cause= # Identifying failure causes would be nice, but difficult
+
+    # Identifying failure causes would be nice, but difficult.
+    # To help diagnose the tracing and our analysis of the traces in
+    # outcome analysis, log the full make command line in the "cause"
+    # column. Once tracing is more mature, I hope this won't be useful
+    # any longer.
+    cause="$*"
+    case "$cause" in
+        *[${nl}\\\;\"]*)
+            # Truncate command lines containing special characters that
+            # we don't want in the outcome file.
+            cause=${cause%%[${nl}\\\;\"]*}...;;
+    esac
 
     for target in $targets; do
         case "$target" in

From ff3ea0ce96401269545064bd9487c5b230be6fe7 Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Thu, 31 Oct 2024 20:04:20 +0100
Subject: [PATCH 10/11] Outcome lines from make: indicate when using a
 different makefile

When emitting a line for the outcome file for `make`, indicate if the target
is being built in a different directory or with a different makefile.

We don't currently do this explicitly with an ambiguous target name, but it
happens under the hood with older versions of CMake (observed with CMake
3.10.2, not with CMake 3.22.1).

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 tests/scripts/all-core.sh |  3 +++
 tests/scripts/quiet/make  | 37 +++++++++++++++++++++++++++++++++----
 2 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/tests/scripts/all-core.sh b/tests/scripts/all-core.sh
index dcca8116d066..f78f762afc20 100644
--- a/tests/scripts/all-core.sh
+++ b/tests/scripts/all-core.sh
@@ -212,6 +212,9 @@ pre_initialize_variables () {
     # Specify character collation for regular expressions and sorting with C locale
     export LC_COLLATE=C
 
+    # Make the location of the root directory available to reporting scripts
+    export MBEDTLS_TEST_ROOT="$PWD"
+
     : ${MBEDTLS_TEST_OUTCOME_FILE=}
     : ${MBEDTLS_TEST_PLATFORM="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"}
     export MBEDTLS_TEST_OUTCOME_FILE
diff --git a/tests/scripts/quiet/make b/tests/scripts/quiet/make
index b31389122ff2..43950a3ef78d 100755
--- a/tests/scripts/quiet/make
+++ b/tests/scripts/quiet/make
@@ -23,21 +23,36 @@ nl='
 ' # set the variable to a newline
 
 log_outcome () {
+    # Options passed to make indicating that a different makefile is used
+    modifier=
+    # Space-separated list of targets
     targets=
+    # Temporary variable used in the loop after an option that takes an argument
     skip=
+
     for arg in "$@"; do
         if [ -n "$skip" ]; then
+            if [ "$skip" = "modifier" ]; then
+                modifier="$modifier$arg "
+            fi
             skip=
             continue
         fi
+
         case $arg in
-            --question|-q) # do-nothing option
+            --dry-run|--help|--just-print|--question|--recon|--version|-n|-q|-v)
+                # do-nothing option
                 return;;
-            --assume-new|--assume-old|--directory|--file| \
+            --directory=*|--file=*|-C?*|-f?*) # Modifier option with its argument
+                modifier="$modifier$arg ";;
+            --directory|--file|-C|-f) # Modifier option with separate argument
+                skip=modifier
+                modifier="$modifier$arg ";;
+            --assume-new|--assume-old| \
             --include-dir|--makefile|--new-file|--old-file|--what-if| \
-            -C|-I|-W|-f|-k|-o) # Option with separate argument
+            -I|-W|-k|-o) # Boring option with separate argument
                 skip=1; continue;;
-            -*) continue;; # Option
+            -*) continue;; # Other option (assumed boring)
             *=*) continue;; # Variable assignment
             *[!-+./0-9@A-Z_a-z]*) # Target with problematic character
                 targets="$targets ${arg%%[!-+./0-9@A-Z_a-z]*}...";;
@@ -45,9 +60,23 @@ log_outcome () {
                 targets="$targets $arg";;
         esac
     done
+
+    if [ -n "$MBEDTLS_TEST_ROOT" ] && [ "$MBEDTLS_TEST_ROOT" != "$PWD" ]; then
+        # Record that `make` was run in a different directory.
+        # This is only accurate when run by all.sh (or more generally
+        # if $MBEDTLS_TEST_ROOT is set in the environment).
+        case "$PWD" in
+            "$MBEDTLS_TEST_ROOT/"*) modifier="-C ${PWD#"${MBEDTLS_TEST_ROOT}/"} $modifier";;
+            *) modifier="-C $PWD $modifier";;
+        esac
+    fi
+
     if [ -n "$targets" ]; then
         targets=${targets# }
     else
+        # Assume that the default target is "all". This is true for
+        # the toplevel Makefile and when using CMake, but might not be
+        # true when using a different makefile.
         targets=all
     fi
 

From caccfc43151b10039108339f1282b5c7f1297f44 Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Thu, 31 Oct 2024 20:08:18 +0100
Subject: [PATCH 11/11] Update framework

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 framework | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/framework b/framework
index 60063336977a..6af71b863384 160000
--- a/framework
+++ b/framework
@@ -1 +1 @@
-Subproject commit 60063336977a591958d3f5414d89c492ba76220a
+Subproject commit 6af71b8633849cc58470aa5997bed7c0462b14f9