diff --git a/framework b/framework
index d68446c9da02..6af71b863384 160000
--- a/framework
+++ b/framework
@@ -1 +1 @@
-Subproject commit d68446c9da02e536279a7aaa5a3c9850742ba30c
+Subproject commit 6af71b8633849cc58470aa5997bed7c0462b14f9
diff --git a/tests/scripts/all-core.sh b/tests/scripts/all-core.sh
index 3b5a053073d0..f78f762afc20 100644
--- a/tests/scripts/all-core.sh
+++ b/tests/scripts/all-core.sh
@@ -212,6 +212,9 @@ pre_initialize_variables () {
     # Specify character collation for regular expressions and sorting with C locale
     export LC_COLLATE=C
 
+    # Make the location of the root directory available to reporting scripts
+    export MBEDTLS_TEST_ROOT="$PWD"
+
     : ${MBEDTLS_TEST_OUTCOME_FILE=}
     : ${MBEDTLS_TEST_PLATFORM="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"}
     export MBEDTLS_TEST_OUTCOME_FILE
@@ -790,6 +793,19 @@ pre_prepare_outcome_file () {
     fi
 }
 
+## write_outcome_line SUITE CASE RESULT [CAUSE]
+## Write a line in the outcome file if enabled.
+## Report $MBEDTLS_TEST_PLATFORM, $MBEDTLS_TEST_CONFIGURATION and the
+## supplied arguments.
+write_outcome_line () {
+    if [ -z "$MBEDTLS_TEST_OUTCOME_FILE" ]; then
+        return
+    fi
+    printf '%s;%s;%s;%s;%s;%s\n' >>"$MBEDTLS_TEST_OUTCOME_FILE" \
+           "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \
+           "$1" "$2" "$3" "${4-}"
+}
+
 pre_print_configuration () {
     if [ $QUIET -eq 1 ]; then
         return
@@ -983,6 +999,12 @@ run_component () {
         fi
     fi
 
+    if [ $component_status -eq 0 ]; then
+        write_outcome_line "all.sh" "whole" "PASS"
+    else
+        write_outcome_line "all.sh" "whole" "FAIL" "$component_status"
+    fi
+
     # Restore the build tree to a clean state.
     cleanup
     unset current_component
diff --git a/tests/scripts/components-build-system.sh b/tests/scripts/components-build-system.sh
index 3047e7625201..dccb57dc4ca9 100644
--- a/tests/scripts/components-build-system.sh
+++ b/tests/scripts/components-build-system.sh
@@ -193,15 +193,18 @@ component_build_cmake_custom_config_file () {
     cd "$OUT_OF_SOURCE_DIR"
 
     # Build once to get the generated files (which need an intact config file)
+    MBEDTLS_TEST_CONFIGURATION="$current_component/out_of_tree/default"
     cmake "$MBEDTLS_ROOT_DIR"
     make
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/out_of_tree/MBEDTLS_CONFIG_FILE"
     msg "build: cmake with -DMBEDTLS_CONFIG_FILE"
     scripts/config.py -w full_config.h full
     echo '#error "cmake -DMBEDTLS_CONFIG_FILE is not working."' > "$MBEDTLS_ROOT_DIR/$CONFIG_H"
     cmake -DGEN_FILES=OFF -DMBEDTLS_CONFIG_FILE=full_config.h "$MBEDTLS_ROOT_DIR"
     make
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/out_of_tree/MBEDTLS_USER_CONFIG_FILE"
     msg "build: cmake with -DMBEDTLS_CONFIG_FILE + -DMBEDTLS_USER_CONFIG_FILE"
     # In the user config, disable one feature (for simplicity, pick a feature
     # that nothing else depends on).
@@ -217,6 +220,7 @@ component_build_cmake_custom_config_file () {
     rm -rf "$OUT_OF_SOURCE_DIR"
 
     # Now repeat the test for an in-tree build:
+    MBEDTLS_TEST_CONFIGURATION="$current_component/in_tree/default"
 
     # Restore config for the in-tree test
     mv include/mbedtls_config_in_tree_copy.h "$CONFIG_H"
@@ -225,12 +229,14 @@ component_build_cmake_custom_config_file () {
     cmake .
     make
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/in_tree/MBEDTLS_CONFIG_FILE"
     msg "build: cmake (in-tree) with -DMBEDTLS_CONFIG_FILE"
     scripts/config.py -w full_config.h full
     echo '#error "cmake -DMBEDTLS_CONFIG_FILE is not working."' > "$MBEDTLS_ROOT_DIR/$CONFIG_H"
     cmake -DGEN_FILES=OFF -DMBEDTLS_CONFIG_FILE=full_config.h .
     make
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/in_tree/MBEDTLS_USER_CONFIG_FILE"
     msg "build: cmake (in-tree) with -DMBEDTLS_CONFIG_FILE + -DMBEDTLS_USER_CONFIG_FILE"
     # In the user config, disable one feature (for simplicity, pick a feature
     # that nothing else depends on).
diff --git a/tests/scripts/components-compiler.sh b/tests/scripts/components-compiler.sh
index 5d22735caf74..ea52390f1be8 100644
--- a/tests/scripts/components-compiler.sh
+++ b/tests/scripts/components-compiler.sh
@@ -25,6 +25,7 @@ test_build_opt () {
     info=$1 cc=$2; shift 2
     $cc --version
     for opt in "$@"; do
+          MBEDTLS_TEST_CONFIGURATION="$current_component$opt"
           msg "build/test: $cc $opt, $info" # ~ 30s
           make CC="$cc" CFLAGS="$opt -std=c99 -pedantic -Wall -Wextra -Werror"
           # We're confident enough in compilers to not run _all_ the tests,
@@ -82,6 +83,7 @@ support_test_gcc_earliest_opt () {
 }
 
 component_build_mingw () {
+    MBEDTLS_TEST_CONFIGURATION="$current_component/w64-mingw32/default/static"
     msg "build: Windows cross build - mingw64, make (Link Library)" # ~ 30s
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -Wall -Wextra -maes -msse2 -mpclmul' WINDOWS_BUILD=1 lib programs
 
@@ -89,11 +91,13 @@ component_build_mingw () {
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -maes -msse2 -mpclmul' WINDOWS_BUILD=1 tests
     make WINDOWS_BUILD=1 clean
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/w64-mingw32/default/shared"
     msg "build: Windows cross build - mingw64, make (DLL)" # ~ 30s
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -Wall -Wextra -maes -msse2 -mpclmul' WINDOWS_BUILD=1 SHARED=1 lib programs
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -Wall -Wextra -maes -msse2 -mpclmul' WINDOWS_BUILD=1 SHARED=1 tests
     make WINDOWS_BUILD=1 clean
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/w64-mingw32/no_AESNI/static"
     msg "build: Windows cross build - mingw64, make (Library only, default config without MBEDTLS_AESNI_C)" # ~ 30s
     ./scripts/config.py unset MBEDTLS_AESNI_C #
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 lib
@@ -133,6 +137,7 @@ component_test_zeroize () {
 
     for optimization_flag in -O2 -O3 -Ofast -Os; do
         for compiler in clang gcc; do
+            MBEDTLS_TEST_CONFIGURATION="$current_component/$optimization_flag/$compiler"
             msg "test: $compiler $optimization_flag, mbedtls_platform_zeroize()"
             make programs CC="$compiler" DEBUG=1 CFLAGS="$optimization_flag"
             gdb -ex "$gdb_disable_aslr" -x tests/scripts/test_zeroize.gdb -nw -batch -nx 2>&1 | tee test_zeroize.log
diff --git a/tests/scripts/components-configuration-crypto.sh b/tests/scripts/components-configuration-crypto.sh
index 6ee0f919d700..9c95ebece81c 100644
--- a/tests/scripts/components-configuration-crypto.sh
+++ b/tests/scripts/components-configuration-crypto.sh
@@ -1448,7 +1448,6 @@ component_test_tfm_config_p256m_driver_accel_ec () {
 
     common_tfm_config
 
-    # Build crypto library
     make CC=$ASAN_CC CFLAGS="$ASAN_CFLAGS -I../tests/include/spe" LDFLAGS="$ASAN_CFLAGS"
 
     # Make sure any built-in EC alg was not re-enabled by accident (additive config)
@@ -1483,7 +1482,7 @@ component_test_tfm_config_no_p256m () {
     echo "#undef MBEDTLS_PSA_P256M_DRIVER_ENABLED" >> "$CONFIG_H"
 
     msg "build: TF-M config without p256m"
-    make CFLAGS='-Werror -Wall -Wextra -I../tests/include/spe' tests
+    make CFLAGS='-Werror -Wall -Wextra -I../tests/include/spe'
 
     # Check that p256m was not built
     not grep p256_ecdsa_ library/libmbedcrypto.a
@@ -2283,6 +2282,7 @@ component_build_aes_variations () {
     # Test that all the combinations build cleanly.
 
     MBEDTLS_ROOT_DIR="$PWD"
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both_directions"
     msg "build: aes.o for all combinations of relevant config options"
 
     build_test_config_combos ${BUILTIN_SRC_PATH}/aes.o validate_aes_config_variations \
@@ -2291,6 +2291,8 @@ component_build_aes_variations () {
         "MBEDTLS_AESNI_C" "MBEDTLS_AESCE_C" "MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH"
 
     cd "$MBEDTLS_ROOT_DIR"
+
+    MBEDTLS_TEST_CONFIGURATION="$current_component/no_decrypt"
     msg "build: aes.o for all combinations of relevant config options + BLOCK_CIPHER_NO_DECRYPT"
 
     # MBEDTLS_BLOCK_CIPHER_NO_DECRYPT is incompatible with ECB in PSA, CBC/XTS/NIST_KW/DES,
@@ -2336,11 +2338,13 @@ END
         #define PSA_WANT_ALG_SHA3_512  1
 END
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/unrolled"
     msg "all loops unrolled"
     make clean
     make -C tests ../tf-psa-crypto/tests/test_suite_shax CFLAGS="-DMBEDTLS_SHA3_THETA_UNROLL=1 -DMBEDTLS_SHA3_PI_UNROLL=1 -DMBEDTLS_SHA3_CHI_UNROLL=1 -DMBEDTLS_SHA3_RHO_UNROLL=1"
     ./tf-psa-crypto/tests/test_suite_shax
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/rolled"
     msg "all loops rolled up"
     make clean
     make -C tests ../tf-psa-crypto/tests/test_suite_shax CFLAGS="-DMBEDTLS_SHA3_THETA_UNROLL=0 -DMBEDTLS_SHA3_PI_UNROLL=0 -DMBEDTLS_SHA3_CHI_UNROLL=0 -DMBEDTLS_SHA3_RHO_UNROLL=0"
@@ -2368,11 +2372,13 @@ component_build_aes_aesce_armcc () {
     scripts/config.py unset MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT
     scripts/config.py set MBEDTLS_HAVE_ASM
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both"
     msg "AESCE, build with default configuration."
     scripts/config.py set MBEDTLS_AESCE_C
     scripts/config.py unset MBEDTLS_AES_USE_HARDWARE_ONLY
     helper_armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8-a+crypto"
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware"
     msg "AESCE, build AESCE only"
     scripts/config.py set MBEDTLS_AESCE_C
     scripts/config.py set MBEDTLS_AES_USE_HARDWARE_ONLY
@@ -2536,20 +2542,24 @@ component_test_block_cipher_no_decrypt_aesni () {
     config_block_cipher_no_decrypt
 
     # test AESNI intrinsics
+    MBEDTLS_TEST_CONFIGURATION="$current_component/AESNI/intrinsics"
     helper_block_cipher_no_decrypt_build_test \
         -s "MBEDTLS_AESNI_C" \
         -c "-mpclmul -msse2 -maes"
 
     # test AESNI assembly
+    MBEDTLS_TEST_CONFIGURATION="$current_component/AESNI/assembly"
     helper_block_cipher_no_decrypt_build_test \
         -s "MBEDTLS_AESNI_C" \
         -c "-mno-pclmul -mno-sse2 -mno-aes"
 
     # test AES C implementation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/software"
     helper_block_cipher_no_decrypt_build_test \
         -u "MBEDTLS_AESNI_C"
 
     # test AESNI intrinsics for i386 target
+    MBEDTLS_TEST_CONFIGURATION="$current_component/AESNI/intrinsics/m32"
     helper_block_cipher_no_decrypt_build_test \
         -s "MBEDTLS_AESNI_C" \
         -c "-m32 -mpclmul -msse2 -maes" \
@@ -2671,6 +2681,7 @@ component_test_psa_crypto_drivers () {
 }
 
 component_build_psa_config_file () {
+    MBEDTLS_TEST_CONFIGURATION="$current_component/MBEDTLS_PSA_CRYPTO_CONFIG_FILE"
     msg "build: make with MBEDTLS_PSA_CRYPTO_CONFIG_FILE" # ~40s
     scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG
     cp "$CRYPTO_CONFIG_H" psa_test_config.h
@@ -2680,6 +2691,7 @@ component_build_psa_config_file () {
     programs/test/query_compile_time_config MBEDTLS_CMAC_C
     make clean
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/MBEDTLS_PSA_CRYPTO_USER_CONFIG_FILE"
     msg "build: make with MBEDTLS_PSA_CRYPTO_CONFIG_FILE + MBEDTLS_PSA_CRYPTO_USER_CONFIG_FILE" # ~40s
     # In the user config, disable one feature and its dependencies, which will
     # reflect on the mbedtls configuration so we can query it with
diff --git a/tests/scripts/components-configuration-tls.sh b/tests/scripts/components-configuration-tls.sh
index b8834d60951e..e01a5989df67 100644
--- a/tests/scripts/components-configuration-tls.sh
+++ b/tests/scripts/components-configuration-tls.sh
@@ -184,39 +184,6 @@ component_test_config_thread () {
     tests/ssl-opt.sh -f 'ECJPAKE.*nolog'
 }
 
-# We're not aware of any other (open source) implementation of EC J-PAKE in TLS
-# that we could use for interop testing. However, we now have sort of two
-# implementations ourselves: one using PSA, the other not. At least test that
-# these two interoperate with each other.
-component_test_tls1_2_ecjpake_compatibility () {
-    msg "build: TLS1.2 server+client w/ EC-JPAKE w/o USE_PSA"
-    scripts/config.py set MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED
-    # Explicitly make lib first to avoid a race condition:
-    # https://github.com/Mbed-TLS/mbedtls/issues/8229
-    make lib
-    make -C programs ssl/ssl_server2 ssl/ssl_client2
-    cp programs/ssl/ssl_server2 s2_no_use_psa
-    cp programs/ssl/ssl_client2 c2_no_use_psa
-
-    msg "build: TLS1.2 server+client w/ EC-JPAKE w/ USE_PSA"
-    scripts/config.py set MBEDTLS_USE_PSA_CRYPTO
-    make clean
-    make lib
-    make -C programs ssl/ssl_server2 ssl/ssl_client2
-    make -C programs test/udp_proxy test/query_compile_time_config
-
-    msg "test: server w/o USE_PSA - client w/ USE_PSA, text password"
-    P_SRV=../s2_no_use_psa tests/ssl-opt.sh -f "ECJPAKE: working, TLS"
-    msg "test: server w/o USE_PSA - client w/ USE_PSA, opaque password"
-    P_SRV=../s2_no_use_psa tests/ssl-opt.sh -f "ECJPAKE: opaque password client only, working, TLS"
-    msg "test: client w/o USE_PSA - server w/ USE_PSA, text password"
-    P_CLI=../c2_no_use_psa tests/ssl-opt.sh -f "ECJPAKE: working, TLS"
-    msg "test: client w/o USE_PSA - server w/ USE_PSA, opaque password"
-    P_CLI=../c2_no_use_psa tests/ssl-opt.sh -f "ECJPAKE: opaque password server only, working, TLS"
-
-    rm s2_no_use_psa c2_no_use_psa
-}
-
 component_test_tls1_2_ccm_psk () {
     msg "build: configs/config-ccm-psk-tls1_2.h"
     cp configs/config-ccm-psk-tls1_2.h "$CONFIG_H"
diff --git a/tests/scripts/components-configuration.sh b/tests/scripts/components-configuration.sh
index 683ac848874a..c4cdbd95c952 100644
--- a/tests/scripts/components-configuration.sh
+++ b/tests/scripts/components-configuration.sh
@@ -237,9 +237,11 @@ component_build_tfm () {
     # configs/config-tfm.h, tested via test-ref-configs.pl.
     cp configs/config-tfm.h "$CONFIG_H"
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/thumb2/clang"
     msg "build: TF-M config, clang, armv7-m thumb2"
     make lib CC="clang" CFLAGS="--target=arm-linux-gnueabihf -march=armv7-m -mthumb -Os -std=c99 -Werror -Wall -Wextra -Wwrite-strings -Wpointer-arith -Wimplicit-fallthrough -Wshadow -Wvla -Wformat=2 -Wno-format-nonliteral -Wshadow -Wasm-operand-widths -Wunused -I../tests/include/spe"
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/native/gcc"
     msg "build: TF-M config, gcc native build"
     make clean
     make lib CC="gcc" CFLAGS="-Os -std=c99 -Werror -Wall -Wextra -Wwrite-strings -Wpointer-arith -Wshadow -Wvla -Wformat=2 -Wno-format-nonliteral -Wshadow -Wformat-signedness -Wlogical-op -I../tests/include/spe"
@@ -287,6 +289,7 @@ component_test_no_platform () {
 }
 
 component_build_mbedtls_config_file () {
+    MBEDTLS_TEST_CONFIGURATION="$current_component/MBEDTLS_PSA_CRYPTO_CONFIG_FILE"
     msg "build: make with MBEDTLS_CONFIG_FILE" # ~40s
     scripts/config.py -w full_config.h full
     echo '#error "MBEDTLS_CONFIG_FILE is not working"' >"$CONFIG_H"
@@ -295,6 +298,7 @@ component_build_mbedtls_config_file () {
     programs/test/query_compile_time_config MBEDTLS_NIST_KW_C
     make clean
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/MBEDTLS_USER_CONFIG_FILE"
     msg "build: make with MBEDTLS_CONFIG_FILE + MBEDTLS_USER_CONFIG_FILE"
     # In the user config, disable one feature (for simplicity, pick a feature
     # that nothing else depends on).
diff --git a/tests/scripts/components-platform.sh b/tests/scripts/components-platform.sh
index a8c8c7befc88..03ad7d7e44dc 100644
--- a/tests/scripts/components-platform.sh
+++ b/tests/scripts/components-platform.sh
@@ -111,12 +111,14 @@ component_test_aesni () { # ~ 60s
     # AESNI detection will fallback to the plain C implementation, so the tests will instead
     # exercise the plain C impl).
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/default"
     msg "build: default config with different AES implementations"
     scripts/config.py set MBEDTLS_AESNI_C
     scripts/config.py unset MBEDTLS_AES_USE_HARDWARE_ONLY
     scripts/config.py set MBEDTLS_HAVE_ASM
 
     # test the intrinsics implementation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/intrinsics"
     msg "AES tests, test intrinsics"
     make clean
     make CC=gcc CFLAGS='-Werror -Wall -Wextra -mpclmul -msse2 -maes'
@@ -124,6 +126,7 @@ component_test_aesni () { # ~ 60s
     ./programs/test/selftest aes | grep "AESNI code" | grep -q "intrinsics"
 
     # test the asm implementation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/assembly"
     msg "AES tests, test assembly"
     make clean
     make CC=gcc CFLAGS='-Werror -Wall -Wextra -mno-pclmul -mno-sse2 -mno-aes'
@@ -131,6 +134,7 @@ component_test_aesni () { # ~ 60s
     ./programs/test/selftest aes | grep "AESNI code" | grep -q "assembly"
 
     # test the plain C implementation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/software"
     scripts/config.py unset MBEDTLS_AESNI_C
     scripts/config.py unset MBEDTLS_AES_USE_HARDWARE_ONLY
     msg "AES tests, plain C"
@@ -142,6 +146,7 @@ component_test_aesni () { # ~ 60s
     grep -q "AES note: built-in implementation." ./programs/test/selftest
 
     # test the intrinsics implementation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/intrinsics"
     scripts/config.py set MBEDTLS_AESNI_C
     scripts/config.py set MBEDTLS_AES_USE_HARDWARE_ONLY
     msg "AES tests, test AESNI only"
@@ -168,6 +173,7 @@ component_test_aesni_m32 () { # ~ 60s
     scripts/config.py set MBEDTLS_HAVE_ASM
 
     # test the intrinsics implementation with gcc
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/intrinsics"
     msg "AES tests, test intrinsics (gcc)"
     make clean
     make CC=gcc CFLAGS='-m32 -Werror -Wall -Wextra' LDFLAGS='-m32'
@@ -177,6 +183,7 @@ component_test_aesni_m32 () { # ~ 60s
     grep -q "AES note: built-in implementation." ./programs/test/selftest
     grep -q mbedtls_aesni_has_support ./programs/test/selftest
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/intrinsics"
     scripts/config.py set MBEDTLS_AESNI_C
     scripts/config.py set MBEDTLS_AES_USE_HARDWARE_ONLY
     msg "AES tests, test AESNI only"
@@ -221,16 +228,19 @@ component_build_aes_armce () {
     scripts/config.py set MBEDTLS_AESCE_C
     scripts/config.py set MBEDTLS_AES_USE_HARDWARE_ONLY
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/aarch64"
     msg "MBEDTLS_AES_USE_HARDWARE_ONLY, clang, aarch64"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=aarch64-linux-gnu -march=armv8-a+crypto"
     msg "clang, test aarch64 crypto instructions built"
     grep -E 'aes[a-z]+\s*[qv]' ${BUILTIN_SRC_PATH}/aesce.s
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/arm"
     msg "MBEDTLS_AES_USE_HARDWARE_ONLY, clang, arm"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a72+crypto -marm"
     msg "clang, test A32 crypto instructions built"
     grep -E 'aes[0-9a-z]+.[0-9]\s*[qv]' ${BUILTIN_SRC_PATH}/aesce.s
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/thumb"
     msg "MBEDTLS_AES_USE_HARDWARE_ONLY, clang, thumb"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a32+crypto -mthumb"
     msg "clang, test T32 crypto instructions built"
@@ -238,16 +248,19 @@ component_build_aes_armce () {
 
     scripts/config.py unset MBEDTLS_AES_USE_HARDWARE_ONLY
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/aarch64"
     msg "MBEDTLS_AES_USE_both, clang, aarch64"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=aarch64-linux-gnu -march=armv8-a+crypto"
     msg "clang, test aarch64 crypto instructions built"
     grep -E 'aes[a-z]+\s*[qv]' ${BUILTIN_SRC_PATH}/aesce.s
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/arm"
     msg "MBEDTLS_AES_USE_both, clang, arm"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a72+crypto -marm"
     msg "clang, test A32 crypto instructions built"
     grep -E 'aes[0-9a-z]+.[0-9]\s*[qv]' ${BUILTIN_SRC_PATH}/aesce.s
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both/thumb"
     msg "MBEDTLS_AES_USE_both, clang, thumb"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a32+crypto -mthumb"
     msg "clang, test T32 crypto instructions built"
@@ -255,16 +268,19 @@ component_build_aes_armce () {
 
     scripts/config.py unset MBEDTLS_AESCE_C
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/software_only/aarch64"
     msg "no MBEDTLS_AESCE_C, clang, aarch64"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=aarch64-linux-gnu -march=armv8-a"
     msg "clang, test aarch64 crypto instructions not built"
     not grep -E 'aes[a-z]+\s*[qv]' ${BUILTIN_SRC_PATH}/aesce.s
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/software_only/arm"
     msg "no MBEDTLS_AESCE_C, clang, arm"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a72 -marm"
     msg "clang, test A32 crypto instructions not built"
     not grep -E 'aes[0-9a-z]+.[0-9]\s*[qv]' ${BUILTIN_SRC_PATH}/aesce.s
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/software_only/thumb"
     msg "no MBEDTLS_AESCE_C, clang, thumb"
     make -B library/../${BUILTIN_SRC_PATH}/aesce.o library/../${BUILTIN_SRC_PATH}/aesce.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a32 -mthumb"
     msg "clang, test T32 crypto instructions not built"
@@ -281,11 +297,13 @@ component_build_sha_armce () {
 
     # Test variations of SHA256 Armv8 crypto extensions
     scripts/config.py set MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY
+        MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/aarch64"
         msg "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY clang, aarch64"
         make -B library/../${BUILTIN_SRC_PATH}/sha256.o library/../${BUILTIN_SRC_PATH}/sha256.s CC=clang CFLAGS="--target=aarch64-linux-gnu -march=armv8-a+crypto"
         msg "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY clang, test aarch64 crypto instructions built"
         grep -E 'sha256[a-z0-9]+\s+[qv]' ${BUILTIN_SRC_PATH}/sha256.s
 
+        MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/arm"
         msg "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY clang, arm"
         make -B library/../${BUILTIN_SRC_PATH}/sha256.o library/../${BUILTIN_SRC_PATH}/sha256.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a72+crypto -marm"
         msg "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY clang, test A32 crypto instructions built"
@@ -295,6 +313,7 @@ component_build_sha_armce () {
 
     # test the deprecated form of the config option
     scripts/config.py set MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY
+        MBEDTLS_TEST_CONFIGURATION="$current_component/hardware/thumb"
         msg "MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY clang, thumb"
         make -B library/../${BUILTIN_SRC_PATH}/sha256.o library/../${BUILTIN_SRC_PATH}/sha256.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a32+crypto -mthumb"
         msg "MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY clang, test T32 crypto instructions built"
@@ -302,6 +321,7 @@ component_build_sha_armce () {
     scripts/config.py unset MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY
 
     scripts/config.py set MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT
+        MBEDTLS_TEST_CONFIGURATION="$current_component/both/aarch64"
         msg "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT clang, aarch64"
         make -B library/../${BUILTIN_SRC_PATH}/sha256.o library/../${BUILTIN_SRC_PATH}/sha256.s CC=clang CFLAGS="--target=aarch64-linux-gnu -march=armv8-a+crypto"
         msg "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT clang, test aarch64 crypto instructions built"
@@ -311,9 +331,11 @@ component_build_sha_armce () {
 
     # test the deprecated form of the config option
     scripts/config.py set MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT
+        MBEDTLS_TEST_CONFIGURATION="$current_component/both/arm"
         msg "MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT clang, arm"
         make -B library/../${BUILTIN_SRC_PATH}/sha256.o library/../${BUILTIN_SRC_PATH}/sha256.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a72+crypto -marm -std=c99"
 
+        MBEDTLS_TEST_CONFIGURATION="$current_component/both/thumb"
         msg "MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT clang, thumb"
         make -B library/../${BUILTIN_SRC_PATH}/sha256.o library/../${BUILTIN_SRC_PATH}/sha256.s CC=clang CFLAGS="--target=arm-linux-gnueabihf -mcpu=cortex-a32+crypto -mthumb"
         msg "MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT clang, test T32 crypto instructions built"
@@ -533,15 +555,18 @@ component_build_arm_clang_thumb () {
 
     scripts/config.py baremetal
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/thumb2/-Os"
     msg "build: clang thumb 2, make"
     make clean
     make CC="clang" CFLAGS='-std=c99 -Werror -Os --target=arm-linux-gnueabihf -march=armv7-m -mthumb' lib
 
     # Some Thumb 1 asm is sensitive to optimisation level, so test both -O0 and -Os
+    MBEDTLS_TEST_CONFIGURATION="$current_component/thumb/-O0"
     msg "build: clang thumb 1 -O0, make"
     make clean
     make CC="clang" CFLAGS='-std=c99 -Werror -O0 --target=arm-linux-gnueabihf -mcpu=arm1136j-s -mthumb' lib
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/thumb/-Os"
     msg "build: clang thumb 1 -Os, make"
     make clean
     make CC="clang" CFLAGS='-std=c99 -Werror -Os --target=arm-linux-gnueabihf -mcpu=arm1136j-s -mthumb' lib
@@ -576,24 +601,31 @@ component_build_armcc () {
     # Compile mostly with -O1 since some Arm inline assembly is disabled for -O0.
 
     # ARM Compiler 6 - Target ARMv7-A
+    MBEDTLS_TEST_CONFIGURATION="$current_component/v7a/-O1"
     helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv7-a"
 
     # ARM Compiler 6 - Target ARMv7-M
+    MBEDTLS_TEST_CONFIGURATION="$current_component/v7m/-O1"
     helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv7-m"
 
     # ARM Compiler 6 - Target ARMv7-M+DSP
+    MBEDTLS_TEST_CONFIGURATION="$current_component/v7m+dsp/-O1"
     helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv7-m+dsp"
 
     # ARM Compiler 6 - Target ARMv8-A - AArch32
+    MBEDTLS_TEST_CONFIGURATION="$current_component/v82a/-O1"
     helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv8.2-a"
 
     # ARM Compiler 6 - Target ARMv8-M
+    MBEDTLS_TEST_CONFIGURATION="$current_component/v8m/-O1"
     helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv8-m.main"
 
     # ARM Compiler 6 - Target Cortex-M0 - no optimisation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/m0/-O0"
     helper_armc6_build_test "-O0 --target=arm-arm-none-eabi -mcpu=cortex-m0"
 
     # ARM Compiler 6 - Target Cortex-M0
+    MBEDTLS_TEST_CONFIGURATION="$current_component/m0/-Os"
     helper_armc6_build_test "-Os --target=arm-arm-none-eabi -mcpu=cortex-m0"
 
     # ARM Compiler 6 - Target ARMv8.2-A - AArch64
@@ -601,6 +633,7 @@ component_build_armcc () {
     # Re-enable MBEDTLS_AESCE_C as this should be supported by the version of armclang
     # that we have in our CI
     scripts/config.py set MBEDTLS_AESCE_C
+    MBEDTLS_TEST_CONFIGURATION="$current_component/v82a+crypto/-O1"
     helper_armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8.2-a+crypto"
 }
 
diff --git a/tests/scripts/quiet/make b/tests/scripts/quiet/make
index 920e5b875fd1..43950a3ef78d 100755
--- a/tests/scripts/quiet/make
+++ b/tests/scripts/quiet/make
@@ -17,3 +17,105 @@ NO_SILENCE=" --version | test "
 TOOL="make"
 
 . "$(dirname "$0")/quiet.sh"
+EXIT_STATUS=$?
+
+nl='
+' # set the variable to a newline
+
+log_outcome () {
+    # Options passed to make indicating that a different makefile is used
+    modifier=
+    # Space-separated list of targets
+    targets=
+    # Temporary variable used in the loop after an option that takes an argument
+    skip=
+
+    for arg in "$@"; do
+        if [ -n "$skip" ]; then
+            if [ "$skip" = "modifier" ]; then
+                modifier="$modifier$arg "
+            fi
+            skip=
+            continue
+        fi
+
+        case $arg in
+            --dry-run|--help|--just-print|--question|--recon|--version|-n|-q|-v)
+                # do-nothing option
+                return;;
+            --directory=*|--file=*|-C?*|-f?*) # Modifier option with its argument
+                modifier="$modifier$arg ";;
+            --directory|--file|-C|-f) # Modifier option with separate argument
+                skip=modifier
+                modifier="$modifier$arg ";;
+            --assume-new|--assume-old| \
+            --include-dir|--makefile|--new-file|--old-file|--what-if| \
+            -I|-W|-k|-o) # Boring option with separate argument
+                skip=1; continue;;
+            -*) continue;; # Other option (assumed boring)
+            *=*) continue;; # Variable assignment
+            *[!-+./0-9@A-Z_a-z]*) # Target with problematic character
+                targets="$targets ${arg%%[!-+./0-9@A-Z_a-z]*}...";;
+            *) # Normal target
+                targets="$targets $arg";;
+        esac
+    done
+
+    if [ -n "$MBEDTLS_TEST_ROOT" ] && [ "$MBEDTLS_TEST_ROOT" != "$PWD" ]; then
+        # Record that `make` was run in a different directory.
+        # This is only accurate when run by all.sh (or more generally
+        # if $MBEDTLS_TEST_ROOT is set in the environment).
+        case "$PWD" in
+            "$MBEDTLS_TEST_ROOT/"*) modifier="-C ${PWD#"${MBEDTLS_TEST_ROOT}/"} $modifier";;
+            *) modifier="-C $PWD $modifier";;
+        esac
+    fi
+
+    if [ -n "$targets" ]; then
+        targets=${targets# }
+    else
+        # Assume that the default target is "all". This is true for
+        # the toplevel Makefile and when using CMake, but might not be
+        # true when using a different makefile.
+        targets=all
+    fi
+
+    # We have a single pass/fail status. This is not accurate when there are
+    # multiple targets: it's possible that some passed and some failed.
+    # To figure out which targets passed when the overall result is a failure,
+    # we'd have to do some complex parsing of logs.
+    if [ $EXIT_STATUS -eq 0 ]; then
+        result=PASS
+    else
+        result=FAIL
+    fi
+
+    # Identifying failure causes would be nice, but difficult.
+    # To help diagnose the tracing and our analysis of the traces in
+    # outcome analysis, log the full make command line in the "cause"
+    # column. Once tracing is more mature, I hope this won't be useful
+    # any longer.
+    cause="$*"
+    case "$cause" in
+        *[${nl}\\\;\"]*)
+            # Truncate command lines containing special characters that
+            # we don't want in the outcome file.
+            cause=${cause%%[${nl}\\\;\"]*}...;;
+    esac
+
+    for target in $targets; do
+        case "$target" in
+            "clean"|"neat") continue;; # Boring
+            cmTC_*) continue;; # Weirdness from CMake
+        esac
+        echo >>"${MBEDTLS_TEST_OUTCOME_FILE}" \
+             "${MBEDTLS_TEST_PLATFORM};${MBEDTLS_TEST_CONFIGURATION};${TOOL};${target};${result};${cause}"
+    done
+}
+
+if [ -n "${MBEDTLS_TEST_OUTCOME_FILE}" ] &&
+   [ -n "${MBEDTLS_TEST_CONFIGURATION}" ]; then
+    log_outcome "$@"
+fi
+
+exit $EXIT_STATUS
diff --git a/tests/scripts/quiet/quiet.sh b/tests/scripts/quiet/quiet.sh
index 0f26184d0d77..e81243d2bf40 100644
--- a/tests/scripts/quiet/quiet.sh
+++ b/tests/scripts/quiet/quiet.sh
@@ -59,7 +59,7 @@ fi
 
 if [[ " $@ " =~ $NO_SILENCE || -n "${VERBOSE_LOGS}" ]]; then
     # Run original command with no output supression
-    exec "${ORIGINAL_TOOL}" "$@"
+    "${ORIGINAL_TOOL}" "$@"
 else
     # Run original command and capture output & exit status
     TMPFILE=$(mktemp "quiet-${TOOL}.XXXXXX")
@@ -75,5 +75,6 @@ else
     rm "${TMPFILE}"
 
     # Propagate the exit status
-    exit $EXIT_STATUS
+    set_status () { return $1; }
+    set_status $EXIT_STATUS
 fi