[ATfE] Set xfails indirectly instead of modifying test sources (#487)

Certain tests will fail when run with ATfE's compiler and libraries. The
toolchain currently modifies the test sources to change the expected
result, but this requires maintaining downstream patches. The LLVM test
driver, lit, provides options to override the expected result set in the
file through command line and environment variables, which would remove
this burden.

This patch makes use of the `LIT_XFAILS` and `LIT_XFAILS_NOT`
environment variables, which will take precedence over what is set in
the sources.

A new script is provided which will generate a list of tests for a given
project/library variant that are expected to give a different result.
This is largely based on the existing set of downstream patches. This
also has the advantage of allowing xfails to be set for specific
variants, instead of globally. This currently only supports the clang,
compiler-rt and libcxx projects.

In order to prepare the correct environment for lit, a wrapper script is
also provided which will accept files containing test lists to be loaded
into the relevant environment variable. Since the test lists can be
quite long, this prevents the command line getting significantly
bloated.

Author: dcandler

arm-software/embedded/CMakeLists.txt

@@ -360,13 +360,37 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/../shared/cmake/generate_toolchain_id.cmake)
 set(LLVM_EXTERNAL_PROJECTS elf2bin)
 set(LLVM_EXTERNAL_ELF2BIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../shared/elf2bin)
 
+# Use the downstream version of lit, which is just a wrapper for the built one.
+# Pass it the xfail/xfail-not lists.
+set(llvm_xfails_file_path "${CMAKE_CURRENT_BINARY_DIR}/llvm.xfails")
+set(llvm_xfails_not_file_path "${CMAKE_CURRENT_BINARY_DIR}/llvm.upasses")
+# Set LLVM_DEFAULT_EXTERNAL_LIT and LLVM_LIT_ARGS before including the LLVM subdirectory.
+set(LLVM_DEFAULT_EXTERNAL_LIT "${CMAKE_CURRENT_SOURCE_DIR}/arm-runtimes/test-support/lit-wrapper.py")
+set(LLVM_LIT_ARGS "${LLVM_LIT_ARGS} -sv --real-lit-path=${CMAKE_CURRENT_BINARY_DIR}/llvm/bin/llvm-lit --xfails-file=${llvm_xfails_file_path} --xfails-not-file=${llvm_xfails_not_file_path}" CACHE STRING "")
+
 set(llvmproject_src_dir ${CMAKE_CURRENT_SOURCE_DIR}/../..)
 add_subdirectory(
     ${llvmproject_src_dir}/llvm llvm
 )
 
 add_dependencies(check-all check-elf2bin)
 
+# Generate a xfail/xfail-not list before running the tests.
+add_custom_target(
+    llvm-xfail-files
+    COMMAND ${Python3_EXECUTABLE}
+        ${CMAKE_CURRENT_SOURCE_DIR}/arm-runtimes/test-support/xfails.py
+        --project llvm
+        --clang ${LLVM_BINARY_DIR}/bin/clang
+        --xfails_file ${llvm_xfails_file_path}
+        --xfails_not_file ${llvm_xfails_not_file_path}
+    DEPENDS
+        clang
+)
+add_dependencies(check-clang llvm-xfail-files)
+add_dependencies(check-llvm llvm-xfail-files)
+add_dependencies(check-all llvm-xfail-files)
+
 if(LLVM_TOOLCHAIN_C_LIBRARY STREQUAL llvmlibc)
     install(
         FILES
@@ -572,11 +596,6 @@ add_dependencies(
     version_txt
 )
 
-# Set LLVM_DEFAULT_EXTERNAL_LIT to the directory of clang
-# which was build in previous step. This path is not exported
-# by add_subdirectory of llvm project
-set(LLVM_DEFAULT_EXTERNAL_LIT "${LLVM_BINARY_DIR}/bin/llvm-lit")
-
 add_custom_target(check-llvm-toolchain-runtimes)
 add_custom_target(check-${LLVM_TOOLCHAIN_C_LIBRARY})
 add_custom_target(check-compiler-rt)

arm-software/embedded/arm-runtimes/CMakeLists.txt

@@ -271,9 +271,24 @@ if(VARIANT STREQUAL "armv6m_soft_nofp")
 endif()
 
 if(ENABLE_COMPILER_RT_TESTS)
+    # Use the downstream version of lit, which is just a wrapper for the built one.
+    # Pass it the xfail/xfail-not lists.
+    set(compiler_rt_xfails_file_path "${CMAKE_CURRENT_BINARY_DIR}/${VARIANT}_compiler-rt.xfails")
+    set(compiler_rt_xfails_not_file_path "${CMAKE_CURRENT_BINARY_DIR}/${VARIANT}_compiler-rt.upasses")
     set(compiler_rt_test_flags "${lib_compile_flags} ${test_link_flags}")
-    set(compiler_rt_lit_args "${LLVM_LIT_ARGS} --path=${LLVM_BINARY_DIR}/bin --xunit-xml-output=results.junit.xml")
-    set(external_lit_path "${LLVM_BINARY_DIR}/bin/llvm-lit")
+    set(compiler_rt_lit_args "${LLVM_LIT_ARGS} --path=${LLVM_BINARY_DIR}/bin --xunit-xml-output=results.junit.xml --real-lit-path=${LLVM_BINARY_DIR}/bin/llvm-lit --xfails-file=${compiler_rt_xfails_file_path} --xfails-not-file=${compiler_rt_xfails_not_file_path}"
+    )
+    set(external_lit_path "${CMAKE_CURRENT_SOURCE_DIR}/test-support/lit-wrapper.py")
+    add_custom_target(
+        compiler-rt-xfail-files
+        COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/test-support/xfails.py
+            --variant ${VARIANT}
+            --project compiler-rt
+            --libc ${C_LIBRARY}
+            --xfails_file ${compiler_rt_xfails_file_path}
+            --xfails_not_file ${compiler_rt_xfails_not_file_path}
+    )
     set(
         compiler_rt_test_cmake_args
         -DCOMPILER_RT_INCLUDE_TESTS=ON
@@ -366,6 +381,7 @@ if(ENABLE_COMPILER_RT_TESTS)
         check-compiler-rt
         compiler_rt-build
         clib-install
+        compiler-rt-xfail-files
     )
     add_dependencies(check-compiler-rt compiler_rt-check-compiler-rt)
 endif()
@@ -797,7 +813,23 @@ if(ENABLE_CXX_LIBS)
             -DLIBCXX_ENABLE_RTTI=${ENABLE_RTTI}
         )
         if(ENABLE_LIBCXX_TESTS)
-            set(cxxlibs_lit_args "${LLVM_LIT_ARGS} --xunit-xml-output=results.junit.xml")
+            # Use the downstream version of lit, which is just a wrapper for the built one.
+            # Pass it the xfail/xfail-not lists.
+            set(libcxx_xfails_file_path "${CMAKE_CURRENT_BINARY_DIR}/${VARIANT}_libcxx.xfails")
+            set(libcxx_xfails_not_file_path "${CMAKE_CURRENT_BINARY_DIR}/${VARIANT}_libcxx.upasses")
+            set(external_lit_path "${CMAKE_CURRENT_SOURCE_DIR}/test-support/lit-wrapper.py")
+            set(libcxx_lit_args "${LLVM_LIT_ARGS} -sv --xunit-xml-output=results.junit.xml --real-lit-path=${LLVM_BINARY_DIR}/bin/llvm-lit --xfails-file=${libcxx_xfails_file_path} --xfails-not-file=${libcxx_xfails_not_file_path}"
+            )
+            add_custom_target(
+                libcxx-xfail-files
+                COMMAND ${Python3_EXECUTABLE}
+                    ${CMAKE_CURRENT_SOURCE_DIR}/test-support/xfails.py
+                    --variant ${VARIANT}
+                    --project libcxx
+                    --libc ${C_LIBRARY}
+                    --xfails_file ${libcxx_xfails_file_path}
+                    --xfails_not_file ${libcxx_xfails_not_file_path}
+            )
             set(cxxlibs_test_cmake_options
                 -DLIBCXX_TEST_CONFIG=${CMAKE_CURRENT_SOURCE_DIR}/test-support/llvm-libc++-picolibc.cfg.in
                 -DLIBCXX_TEST_PARAMS=executor=${lit_test_executor}
@@ -807,7 +839,8 @@ if(ENABLE_CXX_LIBS)
                 -DLIBUNWIND_TEST_PARAMS=executor=${lit_test_executor}
                 -DRUNTIME_TEST_ARCH_FLAGS=${compile_arch_flags}
                 -DRUNTIME_TEST_LINK_FLAGS=${test_link_flags}
-                -DLLVM_LIT_ARGS=${cxxlibs_lit_args}
+                -DLLVM_LIT_ARGS=${libcxx_lit_args}
+                -DLLVM_EXTERNAL_LIT=${external_lit_path}
             )
         endif()
     elseif(C_LIBRARY STREQUAL llvmlibc)
@@ -940,6 +973,7 @@ if(ENABLE_CXX_LIBS)
                 cxxlibs
                 ${check_target}
                 cxxlibs-install
+                libcxx-xfail-files
             )
             add_dependencies(${check_target} cxxlibs-${check_target})
         endforeach()

arm-software/embedded/arm-runtimes/test-support/lit-wrapper.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: Copyright 2024-2025 Arm Limited and/or its affiliates <[email protected]>
+
+"""This is a wrapper script for LLVM's lit, which takes additional options in
+order to prepare a specific environment for running tests.
+
+The new --xfails-file and --xfails-not-file options are functionally similar
+to the existing --xfails and --xfails-not options, but allow for a test
+list to be loaded from a file instead of on the command line. This is
+preferable when there are large number of tests that need to be specified."""
+
+import argparse
+import os
+import runpy
+import sys
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--xfails-file")
+parser.add_argument("--xfails-not-file")
+parser.add_argument("--real-lit-path", required=True)
+# Only parse the known args above. The rest should be processed the actual lit.
+known_args, unknown_args = parser.parse_known_args()
+
+
+def load_env_var(env_var_name, input_filename):
+    """Load the contents of a file into an environment variable."""
+    if input_filename:
+        if os.path.isfile(input_filename):
+            with open(input_filename, "r", encoding="utf-8") as f:
+                xfail_lines = f.readlines()
+            lit_xfails_list = ";".join([line.strip() for line in xfail_lines])
+            # Load the contents of the file into variables
+            if env_var_name in os.environ:
+                os.environ[env_var_name] = (
+                    os.environ[env_var_name] + ";" + lit_xfails_list
+                )
+            else:
+                os.environ[env_var_name] = lit_xfails_list
+        else:
+            raise FileNotFoundError(
+                f"Error: Expected file '{input_filename}' does not exist."
+            )
+
+
+load_env_var("LIT_XFAIL", known_args.xfails_file)
+load_env_var("LIT_XFAIL_NOT", known_args.xfails_not_file)
+
+# Remove expected custom args from the arg list.
+new_argv = [sys.argv[0]] + unknown_args
+sys.argv = new_argv
+
+# Run lit with the new arg list.
+runpy.run_path(known_args.real_lit_path, run_name="__main__")