Refactor testing rules:

1. If ORC_RT_LLVM_TOOLS_DIR is not defined but LLVM_BINARY_DIR is then the
latter is used to set a value for ORC_RT_LLVM_TOOLS_DIR
("${LLVM_BINARY_DIR}/bin").

2. If ORC_RT_LLVM_TOOLS_DIR is available it is used to find the testing tools
(so far just `FileCheck` and `not`).

3. If the test tools are available then the check-orc-rt target is enabled.

The lit config files have also been cut down, and the unit test updated to
compile and return a passing result.

Author: lhames

orc-rt/CMakeLists.txt

@@ -6,19 +6,20 @@
 
 cmake_minimum_required(VERSION 3.20.0)
 
-option(LLVM_RT_INCLUDE_TESTS "Build orcrt tests." ${LLVM_INCLUDE_TESTS})
-
-
 set(LLVM_COMMON_CMAKE_UTILS ${CMAKE_CURRENT_SOURCE_DIR}/../cmake)
 include(${LLVM_COMMON_CMAKE_UTILS}/Modules/CMakePolicy.cmake
   NO_POLICY_SCOPE)
 
-if (ORC_RT_INCLUDE_TESTS)
-    set(LLVM_INCLUDE_UTILS YES)
-endif()
-
 project(OrcRT LANGUAGES C CXX ASM)
 
+list(INSERT CMAKE_MODULE_PATH 0
+  "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
+  "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules"
+  "${CMAKE_CURRENT_SOURCE_DIR}/../runtimes/cmake/Modules"
+  "${LLVM_COMMON_CMAKE_UTILS}"
+  "${LLVM_COMMON_CMAKE_UTILS}/Modules"
+  )
+
 include(GNUInstallDirs)
 
 #===============================================================================
@@ -29,12 +30,16 @@ option(ORC_RT_INCLUDE_DOCS "Build the ORC-RT documentation." ON)
 option(ORC_RT_ENABLE_ASSERTIONS "Enable assertions independent of build mode." ON)
 option(ORC_RT_ENABLE_PEDANTIC "Compile with pedantic enabled." ON)
 option(ORC_RT_ENABLE_WERROR "Fail and stop if a warning is triggered." OFF)
+option(ORC_RT_INCLUDE_TESTS "Build ORC-RT tests." ${LLVM_INCLUDE_TESTS})
 
 set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to conform to")
 set(CMAKE_CXX_STANDARD_REQUIRED YES)
 set(CMAKE_CXX_EXTENSIONS NO)
 set(CMAKE_FOLDER "orc-rt")
 
+set(ORC_RT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+set(ORC_RT_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
+
 #===============================================================================
 # Setup Source Code
 #===============================================================================
@@ -47,50 +52,7 @@ add_subdirectory(include)
 add_subdirectory(lib)
 add_subdirectory(tools)
 
-if(LLVM_INCLUDE_TESTS)
-    if(NOT TARGET FileCheck)
-        find_program(FILECHECK_EXE
-    NAMES FileCheck
-    HINTS ${LLVM_RT_TOOLS_BINARY_DIR}
-    DOC "FileCheck executable")
-    endif()
-    if(FILECHECK_EXE)
-        add_executable(FileCheck IMPORTED GLOBAL)
-        set_property(TARGET FileCheck PROPERTY IMPORTED_LOCATION "${FILECHECK_EXE}")
-        message(DEBUG "Using FileCheck: ${FILECHECK_EXE}")
-    endif()
-    if(NOT TARGET count)
-        find_program(COUNT_EXE
-            NAMES count
-            HINTS ${LLVM_RT_TOOLS_BINARY_DIR}
-            DOC "count executable")
-    endif()
-    if(COUNT_EXE)
-        add_executable(count IMPORTED GLOBAL)
-        set_property(TARGET count PROPERTY IMPORTED_LOCATION "${COUNT_EXE}")
-        message(DEBUG "Using count: ${COUNT_EXE}")
-    endif()
-    if(NOT TARGET not)
-        find_program(NOT_EXE
-            NAMES not
-            HINTS ${LLVM_RT_TOOLS_BINARY_DIR}
-            DOC "not executable")
-    endif()
-    if(NOT_EXE)
-        add_executable(not IMPORTED GLOBAL)
-        set_property(TARGET not PROPERTY IMPORTED_LOCATION "${NOT_EXE}")
-        message(DEBUG "Using not: ${NOT_EXE}")
-    endif()
-
-    set(ORCRT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
-    set(ORCRT_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
-
+if(ORC_RT_INCLUDE_TESTS)
   add_subdirectory(test)
-  set(UNITTEST_DIR ${LLVM_THIRD_PARTY_DIR}/unittest)
-  if(EXISTS ${UNITTEST_DIR}/googletest/include/gtest/gtest.h)
-    if (NOT TARGET gtest)
-      add_subdirectory(${UNITTEST_DIR} third-party/unittest)
-    endif()
-     add_subdirectory(unittests)
-  endif()
-endif()
+  add_subdirectory(unittests)
+endif()

orc-rt/cmake/OrcRTTesting.cmake

@@ -0,0 +1,36 @@
+# Keep track if we have all dependencies.
+set(ORC_RT_LLVM_TOOLS_AVAILABLE TRUE)
+
+if (NOT DEFINED ORC_RT_LLVM_TOOLS_DIR AND DEFINED LLVM_BINARY_DIR)
+  cmake_path(APPEND ORC_RT_LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}" "bin")
+endif()
+
+if (TARGET utils/llvm-lit/all)
+   list(APPEND utils/llvm-lit/all)
+endif()
+
+# Add dependence on FileCheck.
+if (TARGET FileCheck)
+  list(APPEND ORC_RT_TEST_DEPS FileCheck)
+endif()
+
+find_program(ORC_RT_FILECHECK_EXECUTABLE
+  NAMES FileCheck
+  PATHS ${ORC_RT_LLVM_TOOLS_DIR})
+if (NOT ORC_RT_FILECHECK_EXECUTABLE)
+  message(STATUS "Cannot find FileCheck. Please put it in your PATH, set ORC_RT_FILECHECK_EXECUTABLE to its full path, or point ORC_RT_LLVM_TOOLS_DIR to its directory.")
+  set(ORC_RT_LLVM_TOOLS_AVAILABLE FALSE)
+endif()
+
+# Add dependence on not.
+if (TARGET not)
+  list(APPEND ORC_RT_TEST_DEPS not)
+endif()
+
+find_program(ORC_RT_NOT_EXECUTABLE
+  NAMES not
+  PATHS ${ORC_RT_LLVM_TOOLS_DIR})
+if (NOT ORC_RT_NOT_EXECUTABLE)
+  message(STATUS "Cannot find 'not'. Please put it in your PATH, set ORC_RT_NOT_EXECUTABLE to its full path, or point ORC_RT_LLVM_TOOLS_DIR to its directory.")
+  set(ORC_RT_LLVM_TOOLS_AVAILABLE FALSE)
+endif()

orc-rt/test/CMakeLists.txt

@@ -1,37 +1,40 @@
-configure_lit_site_cfg(
-  ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
-  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
-  MAIN_CONFIG
-  ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
-  )
+include(OrcRTTesting)
 
-list(APPEND ORC_RT_TEST_DEPS
-  FileCheck
-  count
-  not
-  orc-executor
+if (ORC_RT_LLVM_TOOLS_AVAILABLE)
+  configure_lit_site_cfg(
+    ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+    ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
+    MAIN_CONFIG
+    ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
   )
 
-add_custom_target(orc-rt-test-depends DEPENDS ${ORC_RT_TEST_DEPS})
-set_target_properties(orc-rt-test-depends PROPERTIES FOLDER "orc-rt/Tests")
-
-add_lit_testsuite(check-orc-rt "Running the ORCRT regression tests"
-  ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS ${ORC_RT_TEST_DEPS}
+  list(APPEND ORC_RT_TEST_DEPS
+    orc-executor
   )
 
-add_lit_testsuites(ORC-RT ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS ${ORC_RT_TEST_DEPS}
-)
+  add_custom_target(orc-rt-test-depends DEPENDS ${ORC_RT_TEST_DEPS})
+  set_target_properties(orc-rt-test-depends PROPERTIES FOLDER "orc-rt/tests")
+
+  add_lit_testsuite(check-orc-rt "Running the ORC-RT regression tests"
+    ${CMAKE_CURRENT_BINARY_DIR}
+    DEPENDS ${ORC_RT_TEST_DEPS}
+    )
+else()
+  message(WARNING "ORC-RT testing tools missing (see cmake log for details). ORC-RT regression tests disabled.")
+endif()
 
-# Add orc-rt unit tests
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.site.cfg.py.in
-  ${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg
+  ${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg.py
   MAIN_CONFIG
-  ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.cfg.py)
+  ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.cfg.py
+  )
+
+# add_lit_testsuites(ORC-RT ${CMAKE_CURRENT_SOURCE_DIR}
+#   DEPENDS ${ORC_RT_TEST_DEPS}
+# )
 
-add_lit_testsuite(check-orcrt-unit "Running orc-rt unittest suites"
+add_lit_testsuite(check-orc-rt-unit "Running orc-rt unittest suites"
   ${CMAKE_CURRENT_BINARY_DIR}/unit
   EXCLUDE_FROM_CHECK_ALL
   DEPENDS OrcRTUnitTests)

orc-rt/test/lit.cfg.py

@@ -1,63 +1,24 @@
 # -*- Python -*-
 
 import os
-import platform
-import re
-import subprocess
-import tempfile
 
 import lit.formats
 import lit.util
 
 from lit.llvm import llvm_config
 from lit.llvm.subst import ToolSubst
-from lit.llvm.subst import FindTool
 
-
-# name: The name of this test suite.
-config.name = "ORCRT"
-
-# TODO: Consolidate the logic for turning on the internal shell by default for all LLVM test suites.
-# See https://github.com/llvm/llvm-project/issues/106636 for more details.
-#
-# We prefer the lit internal shell which provides a better user experience on failures
-# unless the user explicitly disables it with LIT_USE_INTERNAL_SHELL=0 env var.
-use_lit_shell = True
-lit_shell_env = os.environ.get("LIT_USE_INTERNAL_SHELL")
-if lit_shell_env:
-    use_lit_shell = lit.util.pythonize_bool(lit_shell_env)
-
-config.test_format = lit.formats.ShTest(execute_external=not use_lit_shell)
-
-# suffixes: A list of file extensions to treat as test files.
+config.name = "ORC-RT"
+config.test_format = lit.formats.ShTest()
+config.test_source_root = os.path.dirname(__file__)
+config.test_exec_root = os.path.join(config.orc_rt_obj_root, "test")
 config.suffixes = [
-    ".ll",
-    ".test",
-    ".c",
+    ".test"
 ]
-# test_source_root: The root path where tests are located.
-config.test_source_root = os.path.dirname(__file__)
 
-# test_exec_root: The root path where tests should be run.
-config.test_exec_root = os.path.join(config.orcrt_obj_root, "test")
 llvm_config.with_environment(
     "PATH",
-    os.path.join(config.orcrt_obj_root, "tools", "orc-executor"),
+    os.path.join(config.orc_rt_obj_root, "tools", "orc-executor"),
     append_path=True)
-config.substitutions.append(("%PATH%", config.environment["PATH"]))
-# config.substitutions.append(("%shlibext", config.llvm_shlib_ext))
-config.substitutions.append(("%llvm_src_root", config.llvm_src_root))
-# config.substitutions.append(("%host_cxx", config.host_cxx))
-# config.substitutions.append(("%host_cc", config.host_cc))
-if config.llvm_rt_tools_dir:
-    config.llvm_tools_dir = config.llvm_rt_tools_dir
-llvm_config.use_default_substitutions()
-
 
-# excludes: A list of directories to exclude from the testsuite. The 'Inputs'
-# subdirectories contain auxiliary inputs for various tests in their parent
-# directories.
-config.excludes = [
-    "lit.cfg.py",
-    "lit.site.cfg.py.in",
-]
+llvm_config.use_default_substitutions()

orc-rt/test/lit.site.cfg.py.in

@@ -4,20 +4,13 @@ import sys
 
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.orcrt_obj_root = "@ORCRT_BINARY_DIR@"
-config.llvm_rt_tools_dir = "@LLVM_RT_TOOLS_BINARY_DIR@"
-config.llvm_tools_dir = lit_config.substitute("@LLVM_TOOLS_DIR@")
-config.llvm_libs_dir = lit_config.substitute("@LLVM_LIBS_DIR@")
-config.llvm_shlib_dir = lit_config.substitute("@SHLIBDIR@")
-config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
+config.orc_rt_obj_root = "@ORC_RT_BINARY_DIR@"
 config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
-config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
-config.host_arch = "@HOST_ARCH@"
-config.python_executable = "@Python3_EXECUTABLE@"
-config.targets_to_build = "@ORCRT_TARGETS_TO_BUILD@"
+config.llvm_tools_dir = "@ORC_RT_LLVM_TOOLS_DIR@"
+
 
 import lit.llvm
 lit.llvm.initialize(lit_config, config)
 # Let the main config do the real work.
-lit_config.load_config(config, "@ORCRT_SOURCE_DIR@/test/lit.cfg.py")
+lit_config.load_config(config, "@ORC_RT_SOURCE_DIR@/test/lit.cfg.py")

orc-rt/test/unit/lit.cfg.py

@@ -3,20 +3,22 @@
 # Configuration file for the 'lit' test runner.
 
 import os
-import subprocess
 
 import lit.formats
+import lit.util
+
+from lit.llvm import llvm_config
 
 # name: The name of this test suite.
-config.name = "ORCRT-Unit"
+config.name = "ORC-RT-Unit"
 
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = []
 
 # test_source_root: The root path where tests are located.
 # test_exec_root: The root path where tests should be run.
-config.test_exec_root = os.path.join(config.orcrt_obj_root, "unittests")
+config.test_exec_root = os.path.join(config.orc_rt_obj_root, "unittests")
 config.test_source_root = config.test_exec_root
 
 # testFormat: The test format to use to interpret tests.
-config.test_format = lit.formats.GoogleTest(config.llvm_build_mode, "Tests")
+config.test_format = lit.formats.GoogleTest(config.llvm_build_mode, "Tests")

orc-rt/test/unit/lit.site.cfg.py.in

@@ -1,17 +1,16 @@
 @LIT_SITE_CFG_IN_HEADER@
 
+import sys
+
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = lit_config.substitute("@LLVM_TOOLS_DIR@")
-config.llvm_libs_dir = lit_config.substitute("@LLVM_LIBS_DIR@")
-config.llvm_build_mode = lit_config.substitute("@LLVM_BUILD_MODE@")
-config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.orcrt_obj_root = "@ORCRT_BINARY_DIR@"
+config.orc_rt_obj_root = "@ORC_RT_BINARY_DIR@"
+config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
+config.llvm_tools_dir = "@ORC_RT_LLVM_TOOLS_DIR@"
+
 
 import lit.llvm
 lit.llvm.initialize(lit_config, config)
-
 # Let the main config do the real work.
-lit_config.load_config(config, "@ORCRT_SOURCE_DIR@/test/unit/lit.cfg.py")
+lit_config.load_config(config, "@ORC_RT_SOURCE_DIR@/test/lit.cfg.py")

orc-rt/unittests/CMakeLists.txt

@@ -1,5 +1,11 @@
 add_custom_target(OrcRTUnitTests)
-set_target_properties(OrcRTUnitTests PROPERTIES FOLDER "orcrt/Tests")
+set_target_properties(OrcRTUnitTests PROPERTIES FOLDER "orc-rt/Tests")
+
+if (NOT TARGET llvm_gtest)
+  message(WARNING "orc-rt unittests disabled due to GTest being unavailable; "
+                  "Try LLVM_INSTALL_GTEST=ON for the LLVM build")
+  return ()
+endif ()
 
 function(add_orc_rt_unittest test_dirname)
   add_unittest(OrcRTUnitTests ${test_dirname} ${ARGN})

orc-rt/unittests/init.cpp

@@ -2,5 +2,5 @@
 
 
 TEST(TEST, emptyFuncs) {
-  ASSERT_EQ(True, False);
+  ASSERT_TRUE(true);
 }