From 3efe3b783d26ca643718bc935474f5c786a6cd9c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 30 Sep 2025 16:57:38 +0200 Subject: [PATCH 01/12] Improve docstring following the split Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index aed3a1a1e..9bd9cc34c 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -1,8 +1,12 @@ """Run the PSA Crypto API compliance test suite. -Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF, -then compile and run the test suite. The clone is stored at /psa-arch-tests. -Known defects in either the test suite or mbedtls / TF-PSA-Crypto - identified by their test -number - are ignored, while unexpected failures AND successes are reported as errors, to help + +Clone the psa-arch-tests repo and check out the specified commit. +The clone is stored at /psa-arch-tests. +Check out the commit specified by the calling script and apply patches if needed. +Compile the library and the compliance tests and run the test suite. + +The calling script can specify a list of expected failures. +Unexpected failures and successes are reported as errors, to help keep the list of known defects as up to date as possible. """ From 47cb4858efa59b5eeef9c865e4ae1a03a0fafa83 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 30 Sep 2025 17:00:53 +0200 Subject: [PATCH 02/12] Make patch_directory robust wrt glob characters Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 9bd9cc34c..595ef7931 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -14,7 +14,6 @@ # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later import argparse -import glob import os import re import shutil @@ -30,7 +29,7 @@ #pylint: disable=too-many-branches,too-many-statements,too-many-locals def test_compliance(library_build_dir: str, psa_arch_tests_ref: str, - patch_files: List[str], + patch_files: List[Path], expected_failures: List[int]) -> int: """Check out and run compliance tests. @@ -182,8 +181,7 @@ def main(psa_arch_tests_ref: str, expected_failures_list = expected_failures if args.patch_directory: - patch_file_glob = os.path.join(args.patch_directory, '*.patch') - patch_files = sorted(glob.glob(patch_file_glob)) + patch_files = sorted(Path(args.patch_directory).glob('*.patch')) else: patch_files = [] From 4e75c467434e217527678269d9d19754839311f6 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 30 Sep 2025 17:04:37 +0200 Subject: [PATCH 03/12] Use standard way of specifying a default value for --build-dir Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 595ef7931..8d5938470 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -153,13 +153,13 @@ def main(psa_arch_tests_ref: str, psa_arch_tests_ref: tag or sha to use for the arch-tests. expected_failures: default list of expected failures. """ - build_dir = 'out_of_source_build' default_patch_directory = os.path.join(build_tree.guess_project_root(), 'scripts/data_files/psa-arch-tests') # pylint: disable=invalid-name parser = argparse.ArgumentParser() - parser.add_argument('--build-dir', nargs=1, + parser.add_argument('--build-dir', + default='out_of_source_build', help='path to Mbed TLS / TF-PSA-Crypto build directory') parser.add_argument('--expected-failures', nargs='+', help='''set the list of test codes which are expected to fail @@ -170,9 +170,6 @@ def main(psa_arch_tests_ref: str, help='Directory containing patches (*.patch) to apply to psa-arch-tests') args = parser.parse_args() - if args.build_dir is not None: - build_dir = args.build_dir[0] - if expected_failures is None: expected_failures = [] if args.expected_failures is not None: @@ -185,7 +182,7 @@ def main(psa_arch_tests_ref: str, else: patch_files = [] - sys.exit(test_compliance(build_dir, + sys.exit(test_compliance(args.build_dir, psa_arch_tests_ref, patch_files, expected_failures_list)) From 12564d95666a3f113d05a4e3a2dc31ab477cd27e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 30 Sep 2025 17:35:04 +0200 Subject: [PATCH 04/12] Improve --help Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 8d5938470..be4f7045d 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -157,17 +157,19 @@ def main(psa_arch_tests_ref: str, 'scripts/data_files/psa-arch-tests') # pylint: disable=invalid-name - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--build-dir', default='out_of_source_build', - help='path to Mbed TLS / TF-PSA-Crypto build directory') + help=('path to Mbed TLS / TF-PSA-Crypto build directory ' + '(default: %(default)s)')) parser.add_argument('--expected-failures', nargs='+', help='''set the list of test codes which are expected to fail from the command line. If omitted the list given by EXPECTED_FAILURES (inside the script) is used.''') parser.add_argument('--patch-directory', nargs=1, default=default_patch_directory, - help='Directory containing patches (*.patch) to apply to psa-arch-tests') + help=('Directory containing patches (*.patch) to apply ' + 'to psa-arch-tests (default: %(default)s)')) args = parser.parse_args() if expected_failures is None: From aa9f114c16e23d92bb8d49a451272750ee6e1e67 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 30 Sep 2025 17:35:42 +0200 Subject: [PATCH 05/12] Make psa-arch-tests repo and commit configurable on the command line Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index be4f7045d..1a9c2b651 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -28,6 +28,7 @@ #pylint: disable=too-many-branches,too-many-statements,too-many-locals def test_compliance(library_build_dir: str, + psa_arch_tests_repo: str, psa_arch_tests_ref: str, patch_files: List[Path], expected_failures: List[int]) -> int: @@ -60,7 +61,7 @@ def test_compliance(library_build_dir: str, # Reuse existing local clone subprocess.check_call(['git', 'init']) - subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, psa_arch_tests_ref]) + subprocess.check_call(['git', 'fetch', psa_arch_tests_repo, psa_arch_tests_ref]) subprocess.check_call(['git', 'checkout', '--force', 'FETCH_HEAD']) if patch_files: @@ -170,6 +171,14 @@ def main(psa_arch_tests_ref: str, default=default_patch_directory, help=('Directory containing patches (*.patch) to apply ' 'to psa-arch-tests (default: %(default)s)')) + parser.add_argument('--tests-ref', metavar='REF', + default=psa_arch_tests_ref, + help=('Commit (tag/branch/sha) to use for psa-arch-tests ' + '(default: %(default)s)')) + parser.add_argument('--tests-repo', metavar='URL', + default=PSA_ARCH_TESTS_REPO, + help=('Repository to clone for psa-arch-tests ' + '(default: %(default)s)')) args = parser.parse_args() if expected_failures is None: @@ -185,6 +194,7 @@ def main(psa_arch_tests_ref: str, patch_files = [] sys.exit(test_compliance(args.build_dir, - psa_arch_tests_ref, + args.tests_repo, + args.tests_ref, patch_files, expected_failures_list)) From 5a6c2a4d5f1cf2cac0954d89e2631520f6785074 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 30 Sep 2025 18:00:47 +0200 Subject: [PATCH 06/12] Make it explicit whether to reset psa-arch-tests If given a non-empty ref, check out that ref and apply patches. If given an empty ref, keep whatever is there in the psa-arch-tests directory, without doing a checkout or applying patches. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 1a9c2b651..97fadbb11 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -35,8 +35,10 @@ def test_compliance(library_build_dir: str, """Check out and run compliance tests. library_build_dir: path where our library will be built. - psa_arch_tests_ref: tag or sha to use for the arch-tests. - patch: patch to apply to the arch-tests with ``patch -p1``. + psa_arch_tests_ref: tag or sha to use for the arch-tests + (empty=default/leave alone). + patch: patch to apply to the arch-tests with ``patch -p1`` + (not if psa_arch_tests_ref is empty). expected_failures: default list of expected failures. """ root_dir = os.getcwd() @@ -62,14 +64,16 @@ def test_compliance(library_build_dir: str, # Reuse existing local clone subprocess.check_call(['git', 'init']) subprocess.check_call(['git', 'fetch', psa_arch_tests_repo, psa_arch_tests_ref]) - subprocess.check_call(['git', 'checkout', '--force', 'FETCH_HEAD']) - if patch_files: + # Reuse existing working copy if psa_arch_tests_ref is empty. + # Otherwise check out and patch the specified ref. + if psa_arch_tests_ref: + subprocess.check_call(['git', 'checkout', '--force', 'FETCH_HEAD']) subprocess.check_call(['git', 'reset', '--hard']) - for patch_file in patch_files: - with open(os.path.join(root_dir, patch_file), 'rb') as patch: - subprocess.check_call(['patch', '-p1'], - stdin=patch) + for patch_file in patch_files: + with open(os.path.join(root_dir, patch_file), 'rb') as patch: + subprocess.check_call(['patch', '-p1'], + stdin=patch) build_dir = 'api-tests/build' try: @@ -174,6 +178,7 @@ def main(psa_arch_tests_ref: str, parser.add_argument('--tests-ref', metavar='REF', default=psa_arch_tests_ref, help=('Commit (tag/branch/sha) to use for psa-arch-tests ' + '(empty to use whatever is there and skip patching) ' '(default: %(default)s)')) parser.add_argument('--tests-repo', metavar='URL', default=PSA_ARCH_TESTS_REPO, From c8f1fc9e66bd08757b7925144d2456700d6d5325 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 30 Sep 2025 18:01:42 +0200 Subject: [PATCH 07/12] Save time by only building the library, not our tests Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 97fadbb11..2ef1d9d46 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -47,6 +47,7 @@ def test_compliance(library_build_dir: str, tmp_env['CC'] = 'gcc' subprocess.check_call(['cmake', '.', '-GUnix Makefiles', '-B' + library_build_dir, + '-DENABLE_TESTING=Off', '-DENABLE_PROGRAMS=Off', '-DCMAKE_INSTALL_PREFIX=' + str(install_dir)], env=tmp_env) subprocess.check_call(['cmake', '--build', library_build_dir, '--target', 'install']) From dd5054248dc25bcdedca21cd37772a0e1a9834eb Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 3 Oct 2025 18:31:24 +0200 Subject: [PATCH 08/12] Fix fatal exception on expected failures Use sets to track membership rather than lists. It's more natural. In test_compliance, don't overwrite the mutable list passed by the caller. It's rude. This commit fixes a bug whereby an expected failure would lead to code that raised a `ValueError`, with an exception handler that expected this condition to raise `KeyError`. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 2ef1d9d46..7503fc4ee 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -105,9 +105,9 @@ def test_compliance(library_build_dir: str, '^TEST RESULT: (?PFAILED|PASSED)' ) test = -1 - unexpected_successes = expected_failures.copy() - expected_failures.clear() - unexpected_failures = [] # type: List[int] + unexpected_successes = set(expected_failures) + seen_expected_failures = set() + unexpected_failures = set() if proc.stdout is None: return 1 @@ -120,12 +120,12 @@ def test_compliance(library_build_dir: str, if test_num is not None: test = int(test_num) elif groupdict['test_result'] == 'FAILED': - try: + if test in unexpected_successes: unexpected_successes.remove(test) - expected_failures.append(test) + seen_expected_failures.add(test) print('Expected failure, ignoring') - except KeyError: - unexpected_failures.append(test) + else: + unexpected_failures.add(test) print('ERROR: Unexpected failure') elif test in unexpected_successes: print('ERROR: Unexpected success') @@ -134,7 +134,7 @@ def test_compliance(library_build_dir: str, print() print('***** test_psa_compliance.py report ******') print() - print('Expected failures:', ', '.join(str(i) for i in expected_failures)) + print('Expected failures:', ', '.join(str(i) for i in seen_expected_failures)) print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) print() From eabf862efe458edea693c333cc11466f461a1b45 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 3 Oct 2025 21:57:37 +0200 Subject: [PATCH 09/12] Mix psa-arch-tests stderr with stdout Normally there's nothing on stderr, but if there is something (e.g. an assertion failure from libc or a sanitizer), it will be read and printed in order with respect to normal output. This will make errors easier to debug. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 7503fc4ee..e070f4129 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -98,7 +98,10 @@ def test_compliance(library_build_dir: str, subprocess.check_call(['cmake', '--build', '.']) proc = subprocess.Popen(['./psa-arch-tests-crypto'], - bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) test_re = re.compile( '^TEST: (?P[0-9]*)|' From feccfdce9f38f0a6ab0898d493d858a3ad5add13 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 6 Oct 2025 12:02:35 +0200 Subject: [PATCH 10/12] Sort collections explicitly when printing Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index e070f4129..98fcaee78 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -137,8 +137,8 @@ def test_compliance(library_build_dir: str, print() print('***** test_psa_compliance.py report ******') print() - print('Expected failures:', ', '.join(str(i) for i in seen_expected_failures)) - print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) + print('Expected failures:', ', '.join(str(i) for i in sorted(seen_expected_failures))) + print('Unexpected failures:', ', '.join(str(i) for i in sorted(unexpected_failures))) print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) print() if unexpected_successes or unexpected_failures: From 4c1dc1cb25499a427c26ca029318e3e7b1d520e2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 6 Oct 2025 12:04:56 +0200 Subject: [PATCH 11/12] Update docstring Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 98fcaee78..92498a3a3 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -37,8 +37,8 @@ def test_compliance(library_build_dir: str, library_build_dir: path where our library will be built. psa_arch_tests_ref: tag or sha to use for the arch-tests (empty=default/leave alone). - patch: patch to apply to the arch-tests with ``patch -p1`` - (not if psa_arch_tests_ref is empty). + patch_files: patches (list of file names) to apply to the arch-tests + with ``patch -p1`` (not if psa_arch_tests_ref is empty). expected_failures: default list of expected failures. """ root_dir = os.getcwd() From c40dc756c1b8ab33145b1bdd415717d80c9024d2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 6 Oct 2025 20:09:10 +0200 Subject: [PATCH 12/12] Make the psa-arch-tests build directory configurable This is useful when testing different versions or different build options, or to point to an existing set of worktrees. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 92498a3a3..581eed435 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -26,11 +26,12 @@ PSA_ARCH_TESTS_REPO = 'https://github.com/ARM-software/psa-arch-tests.git' -#pylint: disable=too-many-branches,too-many-statements,too-many-locals +#pylint: disable=too-many-arguments,too-many-branches,too-many-statements,too-many-locals def test_compliance(library_build_dir: str, psa_arch_tests_repo: str, psa_arch_tests_ref: str, patch_files: List[Path], + psa_arch_tests_dir: str, expected_failures: List[int]) -> int: """Check out and run compliance tests. @@ -57,18 +58,18 @@ def test_compliance(library_build_dir: str, else: crypto_library_path = install_dir.joinpath("lib/libtfpsacrypto.a") - psa_arch_tests_dir = 'psa-arch-tests' os.makedirs(psa_arch_tests_dir, exist_ok=True) try: os.chdir(psa_arch_tests_dir) - # Reuse existing local clone - subprocess.check_call(['git', 'init']) - subprocess.check_call(['git', 'fetch', psa_arch_tests_repo, psa_arch_tests_ref]) - # Reuse existing working copy if psa_arch_tests_ref is empty. # Otherwise check out and patch the specified ref. if psa_arch_tests_ref: + # Reuse existing local clone + if not os.path.exists('.git'): + subprocess.check_call(['git', 'init']) + subprocess.check_call(['git', 'fetch', psa_arch_tests_repo, psa_arch_tests_ref]) + subprocess.check_call(['git', 'checkout', '--force', 'FETCH_HEAD']) subprocess.check_call(['git', 'reset', '--hard']) for patch_file in patch_files: @@ -179,6 +180,10 @@ def main(psa_arch_tests_ref: str, default=default_patch_directory, help=('Directory containing patches (*.patch) to apply ' 'to psa-arch-tests (default: %(default)s)')) + parser.add_argument('--tests-dir', metavar='DIR', + default='psa-arch-tests', + help=('path to psa-arch-tests build directory ' + '(default: %(default)s)')) parser.add_argument('--tests-ref', metavar='REF', default=psa_arch_tests_ref, help=('Commit (tag/branch/sha) to use for psa-arch-tests ' @@ -206,4 +211,5 @@ def main(psa_arch_tests_ref: str, args.tests_repo, args.tests_ref, patch_files, + args.tests_dir, expected_failures_list))