From 353416501bcb6dfb84a5816f44549e904e891007 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 18 Jul 2025 17:50:59 +0200 Subject: [PATCH 1/4] New option: make_generated_files.py --clean Clean generated files. In Mbed TLS, `make neat`. is equivalent to `make clean; framework/scripts/make_generated_files.py --clean`. Signed-off-by: Gilles Peskine --- scripts/make_generated_files.py | 42 ++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/scripts/make_generated_files.py b/scripts/make_generated_files.py index 1ca0f2da7..7948225c9 100755 --- a/scripts/make_generated_files.py +++ b/scripts/make_generated_files.py @@ -9,7 +9,9 @@ Generate the TF-PSA-Crypto generated files """ import argparse +import enum import filecmp +import os import shutil import subprocess import sys @@ -219,21 +221,39 @@ def check_generated_files(generation_scripts: List[GenerationScript], root: Path file.unlink() bak_file.rename(file) + +class Action(enum.Enum): + CHECK = 0 + CLEAN = 1 + GENERATE = 2 + LIST = 3 + def main(): """ Main function of this program """ parser = argparse.ArgumentParser() - - parser.add_argument('--list', action='store_true', - default=False, help='List generated files.') parser.add_argument('--root', metavar='DIR', help='Root of the tree containing the generated files \ - to check (default: Mbed TLS or TF-PSA-Cryto root.)') - parser.add_argument('--check', action='store_true', - default=False, help='Check the generated files in root') + (default: Mbed TLS or TF-PSA-Cryto root.)') + + action = parser.add_mutually_exclusive_group() + action.add_argument('--check', + dest='action', action='store_const', const=Action.CHECK, + help='Check that the generated files are up to date.') + action.add_argument('--clean', + dest='action', action='store_const', const=Action.CLEAN, + help='Remove all generated files') + action.add_argument('--generate', + dest='action', action='store_const', const=Action.GENERATE, + help='Generate target-independent file (default mode)') + action.add_argument('--list', + dest='action', action='store_const', const=Action.LIST, + help='List generated files and exit') args = parser.parse_args() + if not args.action: + args.action = Action.GENERATE if not build_tree.looks_like_root("."): raise RuntimeError("This script must be run from Mbed TLS or TF-PSA-Crypto root.") @@ -245,13 +265,19 @@ def main(): else: raise Exception("No support for Mbed TLS 3.6") - if args.list: + if args.action == Action.LIST: files = get_generated_files(generation_scripts) for file in files: print(str(file)) - elif args.check: + elif args.action == Action.CLEAN: + files = get_generated_files(generation_scripts) + for file in files: + if os.path.exists(file): + os.remove(file) + elif args.action == Action.CHECK: check_generated_files(generation_scripts, Path(args.root or ".")) else: + assert args.action == Action.GENERATE make_generated_files(generation_scripts) if __name__ == "__main__": From ff82db616ed977301a976f8f1b36f553a135aa92 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 18 Jul 2025 17:52:43 +0200 Subject: [PATCH 2/4] Pick whether to pre-generate files solely based on the component Whether a component started with configuration-independent platform-independent generated files present or not depended on the repository and on the state of the working directory: * In TF-PSA-Crypto, we started with whatever generated files were present before invoking the script, and didn't touch them between components. * In Mbed TLS, we always started with generated files present (for subsequent components, only if they hadn't been messed up). After this change, the behavior is the same across repositories and doesn't depend on the initial state of the working directory: * `check_xxx` components start with the generated files present. This is either indifferent or necessary depending on the component. (Without generated files, some components such as `check_names` fail, while others such as `check_test_cases` miss stuff that they should check.) * Other components (`build_xxx`, `test_xxx`, etc.) start with the generated files absent. This is generally desirable so that we test that the build scripts generate them as needed. Fix Mbed-TLS/mbedtls-framework#188 Signed-off-by: Gilles Peskine --- scripts/all-core.sh | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/scripts/all-core.sh b/scripts/all-core.sh index 36776a24c..ca5a04d2f 100644 --- a/scripts/all-core.sh +++ b/scripts/all-core.sh @@ -365,6 +365,9 @@ EOF # Does not remove generated source files. cleanup() { + # In Mbed TLS, we do in-tree builds, so clean up. + # In TF-PSA-Crypto, in-tree builds are technically possible but discouraged + # and not done in all.sh so we don't clean up. if in_mbedtls_repo; then command make clean fi @@ -874,17 +877,6 @@ pre_check_tools () { "$@" "${BASH_SOURCE%/*}"/output_env.sh } -pre_generate_files() { - # since make doesn't have proper dependencies, remove any possibly outdate - # file that might be around before generating fresh ones - make neat - if [ $QUIET -eq 1 ]; then - make generated_files >/dev/null - else - make generated_files - fi -} - pre_load_helpers () { # Use a path relative to the currently-sourced file. test_script_dir="${BASH_SOURCE%/*}" @@ -951,6 +943,27 @@ run_component () { pre_create_tf_psa_crypto_out_of_source_directory fi + case ${current_component#component_} in + check_*|tf_psa_crypto_check*) + # Many check_xxx components expect generated files to be present, + # and may silently check less if they aren't. + if in_mbedtls_repo; then + make generated_files + else + $FRAMEWORK/scripts/make_generated_files.py + fi;; + *) + # Build (and build-and-test) components are supposed to work + # whether generated files are already present or not. + # Test with the generated files absent, since if this works, + # it's likely to work with generated files present as well. + if in_mbedtls_repo; then + make neat + else + $FRAMEWORK/scripts/make_generated_files.py --clean + fi + esac + # Run the component in a subshell, with error trapping and output # redirection set up based on the relevant options. if [ $KEEP_GOING -eq 1 ]; then @@ -1021,9 +1034,6 @@ main () { pre_print_configuration pre_check_tools cleanup - if in_mbedtls_repo; then - pre_generate_files - fi # Run the requested tests. for ((error_test_i=1; error_test_i <= error_test; error_test_i++)); do From 6309a08381fdd8a37c85e3d7a37db85bc20de94f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 23 Jul 2025 15:32:53 +0200 Subject: [PATCH 3/4] Pre-generate files in build-only components This is needed for cross-compilation because some scripts to generate generated files compile and execute a C program, and our scripts aren't set up to use a different compiler than the one we cross-compile with. Do this for all build components. It's not ideal but not harmful, and it's easier than figuring out exactly which build components are problematic. Signed-off-by: Gilles Peskine --- scripts/all-core.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/all-core.sh b/scripts/all-core.sh index ca5a04d2f..be366e470 100644 --- a/scripts/all-core.sh +++ b/scripts/all-core.sh @@ -944,16 +944,21 @@ run_component () { fi case ${current_component#component_} in + # - Many check_xxx components expect generated files to be present, + # and may silently check less if they aren't. + # - Build components that cross-compile would fail when trying to + # generate files during the build if they can't an executable + # produced by ${CC}. To keep things simple, pre-generate the files + # for all build-only components, even the ones that don't require it. + build_*|tf_psa_crypto_build*|\ check_*|tf_psa_crypto_check*) - # Many check_xxx components expect generated files to be present, - # and may silently check less if they aren't. if in_mbedtls_repo; then make generated_files else $FRAMEWORK/scripts/make_generated_files.py fi;; *) - # Build (and build-and-test) components are supposed to work + # Other build or build-and-test components are supposed to work # whether generated files are already present or not. # Test with the generated files absent, since if this works, # it's likely to work with generated files present as well. From 876b270c03372cd755e0f2a26009244a781f5c16 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 23 Jul 2025 15:34:26 +0200 Subject: [PATCH 4/4] Pre-generate files in /library in libtestdriver1 components We need this because `library/Makefile` and `libtestdriver1_rewrite.pl` don't set things up properly in `tests/libtestdriver1/library`. The visible symptom is that when not all generated files are present, the makefile looks for `../framework/exported.make` which is not present. I have not gone down this rabbit hole to see how much effort it would be to make it work. Signed-off-by: Gilles Peskine --- scripts/all-helpers.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/all-helpers.sh b/scripts/all-helpers.sh index 5bc6286d8..427d379ef 100644 --- a/scripts/all-helpers.sh +++ b/scripts/all-helpers.sh @@ -103,6 +103,17 @@ helper_libtestdriver1_adjust_config() { # 2. optional: a space-separate list of things to also support. # Here "things" are PSA_WANT_ symbols but with PSA_WANT_ removed. helper_libtestdriver1_make_drivers() { + # Ensure that the configuration-independent platform-independent generated + # files are present in the source tree. We specifically need the ones + # involved in building the crypto library, because the libtestdriver1 + # rewritten library/Makefile can't build them (it doesn't have all + # paths properly rewritten for that). + if in_mbedtls_repo; then + make -C library generated_files + else + $FRAMEWORK/scripts/make_generated_files.py + fi + loc_accel_flags=$( echo "$1 ${2-}" | sed 's/[^ ]* */-DLIBTESTDRIVER1_MBEDTLS_PSA_ACCEL_&/g' ) make CC=$ASAN_CC -C tests libtestdriver1.a CFLAGS=" $ASAN_CFLAGS $loc_accel_flags" LDFLAGS="$ASAN_CFLAGS" }