|
| 1 | +"""Run the PSA Crypto API compliance test suite. |
| 2 | +Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF, |
| 3 | +then compile and run the test suite. The clone is stored at <repository root>/psa-arch-tests. |
| 4 | +Known defects in either the test suite or mbedtls / TF-PSA-Crypto - identified by their test |
| 5 | +number - are ignored, while unexpected failures AND successes are reported as errors, to help |
| 6 | +keep the list of known defects as up to date as possible. |
| 7 | +""" |
| 8 | + |
| 9 | +# Copyright The Mbed TLS Contributors |
| 10 | +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later |
| 11 | + |
| 12 | +import argparse |
| 13 | +import glob |
| 14 | +import os |
| 15 | +import re |
| 16 | +import shutil |
| 17 | +import subprocess |
| 18 | +import sys |
| 19 | +from typing import List, Optional |
| 20 | +from pathlib import Path |
| 21 | + |
| 22 | +from . import build_tree |
| 23 | + |
| 24 | +PSA_ARCH_TESTS_REPO = 'https://github.com/ARM-software/psa-arch-tests.git' |
| 25 | + |
| 26 | +#pylint: disable=too-many-branches,too-many-statements,too-many-locals |
| 27 | +def test_compliance(library_build_dir: str, |
| 28 | + psa_arch_tests_ref: str, |
| 29 | + patch_files: List[str], |
| 30 | + expected_failures: List[int]) -> int: |
| 31 | + """Check out and run compliance tests. |
| 32 | +
|
| 33 | + library_build_dir: path where our library will be built. |
| 34 | + psa_arch_tests_ref: tag or sha to use for the arch-tests. |
| 35 | + patch: patch to apply to the arch-tests with ``patch -p1``. |
| 36 | + expected_failures: default list of expected failures. |
| 37 | + """ |
| 38 | + root_dir = os.getcwd() |
| 39 | + install_dir = Path(library_build_dir + "/install_dir").resolve() |
| 40 | + tmp_env = os.environ |
| 41 | + tmp_env['CC'] = 'gcc' |
| 42 | + subprocess.check_call(['cmake', '.', '-GUnix Makefiles', |
| 43 | + '-B' + library_build_dir, |
| 44 | + '-DCMAKE_INSTALL_PREFIX=' + str(install_dir)], |
| 45 | + env=tmp_env) |
| 46 | + subprocess.check_call(['cmake', '--build', library_build_dir, '--target', 'install']) |
| 47 | + |
| 48 | + if build_tree.is_mbedtls_3_6(): |
| 49 | + crypto_library_path = install_dir.joinpath("lib/libmbedcrypto.a") |
| 50 | + else: |
| 51 | + crypto_library_path = install_dir.joinpath("lib/libtfpsacrypto.a") |
| 52 | + |
| 53 | + psa_arch_tests_dir = 'psa-arch-tests' |
| 54 | + os.makedirs(psa_arch_tests_dir, exist_ok=True) |
| 55 | + try: |
| 56 | + os.chdir(psa_arch_tests_dir) |
| 57 | + |
| 58 | + # Reuse existing local clone |
| 59 | + subprocess.check_call(['git', 'init']) |
| 60 | + subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, psa_arch_tests_ref]) |
| 61 | + subprocess.check_call(['git', 'checkout', '--force', 'FETCH_HEAD']) |
| 62 | + |
| 63 | + if patch_files: |
| 64 | + subprocess.check_call(['git', 'reset', '--hard']) |
| 65 | + for patch_file in patch_files: |
| 66 | + with open(os.path.join(root_dir, patch_file), 'rb') as patch: |
| 67 | + subprocess.check_call(['patch', '-p1'], |
| 68 | + stdin=patch) |
| 69 | + |
| 70 | + build_dir = 'api-tests/build' |
| 71 | + try: |
| 72 | + shutil.rmtree(build_dir) |
| 73 | + except FileNotFoundError: |
| 74 | + pass |
| 75 | + os.mkdir(build_dir) |
| 76 | + os.chdir(build_dir) |
| 77 | + |
| 78 | + #pylint: disable=bad-continuation |
| 79 | + subprocess.check_call([ |
| 80 | + 'cmake', '..', |
| 81 | + '-GUnix Makefiles', |
| 82 | + '-DTARGET=tgt_dev_apis_stdc', |
| 83 | + '-DTOOLCHAIN=HOST_GCC', |
| 84 | + '-DSUITE=CRYPTO', |
| 85 | + '-DPSA_CRYPTO_LIB_FILENAME={}'.format(str(crypto_library_path)), |
| 86 | + '-DPSA_INCLUDE_PATHS=' + str(install_dir.joinpath("include")) |
| 87 | + ]) |
| 88 | + |
| 89 | + subprocess.check_call(['cmake', '--build', '.']) |
| 90 | + |
| 91 | + proc = subprocess.Popen(['./psa-arch-tests-crypto'], |
| 92 | + bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) |
| 93 | + |
| 94 | + test_re = re.compile( |
| 95 | + '^TEST: (?P<test_num>[0-9]*)|' |
| 96 | + '^TEST RESULT: (?P<test_result>FAILED|PASSED)' |
| 97 | + ) |
| 98 | + test = -1 |
| 99 | + unexpected_successes = expected_failures.copy() |
| 100 | + expected_failures.clear() |
| 101 | + unexpected_failures = [] # type: List[int] |
| 102 | + if proc.stdout is None: |
| 103 | + return 1 |
| 104 | + |
| 105 | + for line in proc.stdout: |
| 106 | + print(line, end='') |
| 107 | + match = test_re.match(line) |
| 108 | + if match is not None: |
| 109 | + groupdict = match.groupdict() |
| 110 | + test_num = groupdict['test_num'] |
| 111 | + if test_num is not None: |
| 112 | + test = int(test_num) |
| 113 | + elif groupdict['test_result'] == 'FAILED': |
| 114 | + try: |
| 115 | + unexpected_successes.remove(test) |
| 116 | + expected_failures.append(test) |
| 117 | + print('Expected failure, ignoring') |
| 118 | + except KeyError: |
| 119 | + unexpected_failures.append(test) |
| 120 | + print('ERROR: Unexpected failure') |
| 121 | + elif test in unexpected_successes: |
| 122 | + print('ERROR: Unexpected success') |
| 123 | + proc.wait() |
| 124 | + |
| 125 | + print() |
| 126 | + print('***** test_psa_compliance.py report ******') |
| 127 | + print() |
| 128 | + print('Expected failures:', ', '.join(str(i) for i in expected_failures)) |
| 129 | + print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) |
| 130 | + print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) |
| 131 | + print() |
| 132 | + if unexpected_successes or unexpected_failures: |
| 133 | + if unexpected_successes: |
| 134 | + print('Unexpected successes encountered.') |
| 135 | + print('Please remove the corresponding tests from ' |
| 136 | + 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') |
| 137 | + print() |
| 138 | + print('FAILED') |
| 139 | + return 1 |
| 140 | + else: |
| 141 | + print('SUCCESS') |
| 142 | + return 0 |
| 143 | + finally: |
| 144 | + os.chdir(root_dir) |
| 145 | + |
| 146 | +def main(psa_arch_tests_ref: str, |
| 147 | + expected_failures: Optional[List[int]] = None) -> None: |
| 148 | + """Command line entry point. |
| 149 | +
|
| 150 | + psa_arch_tests_ref: tag or sha to use for the arch-tests. |
| 151 | + expected_failures: default list of expected failures. |
| 152 | + """ |
| 153 | + build_dir = 'out_of_source_build' |
| 154 | + default_patch_directory = os.path.join(build_tree.guess_project_root(), |
| 155 | + 'scripts/data_files/psa-arch-tests') |
| 156 | + |
| 157 | + # pylint: disable=invalid-name |
| 158 | + parser = argparse.ArgumentParser() |
| 159 | + parser.add_argument('--build-dir', nargs=1, |
| 160 | + help='path to Mbed TLS / TF-PSA-Crypto build directory') |
| 161 | + parser.add_argument('--expected-failures', nargs='+', |
| 162 | + help='''set the list of test codes which are expected to fail |
| 163 | + from the command line. If omitted the list given by |
| 164 | + EXPECTED_FAILURES (inside the script) is used.''') |
| 165 | + parser.add_argument('--patch-directory', nargs=1, |
| 166 | + default=default_patch_directory, |
| 167 | + help='Directory containing patches (*.patch) to apply to psa-arch-tests') |
| 168 | + args = parser.parse_args() |
| 169 | + |
| 170 | + if args.build_dir is not None: |
| 171 | + build_dir = args.build_dir[0] |
| 172 | + |
| 173 | + if expected_failures is None: |
| 174 | + expected_failures = [] |
| 175 | + if args.expected_failures is not None: |
| 176 | + expected_failures_list = [int(i) for i in args.expected_failures] |
| 177 | + else: |
| 178 | + expected_failures_list = expected_failures |
| 179 | + |
| 180 | + if args.patch_directory: |
| 181 | + patch_file_glob = os.path.join(args.patch_directory, '*.patch') |
| 182 | + patch_files = sorted(glob.glob(patch_file_glob)) |
| 183 | + else: |
| 184 | + patch_files = [] |
| 185 | + |
| 186 | + sys.exit(test_compliance(build_dir, |
| 187 | + psa_arch_tests_ref, |
| 188 | + patch_files, |
| 189 | + expected_failures_list)) |
0 commit comments