From cc35384bfbe7156a1696a906fd2f90ccc9df8ab2 Mon Sep 17 00:00:00 2001 From: Jake Hillion Date: Mon, 4 Aug 2025 11:49:03 +0100 Subject: [PATCH 1/2] tests: add support for dynamically skipping tests based on llvm version Some tests are useful but are either broken on individual LLVM versions or before/after an LLVM version. Add new options to the test TOML that let us skip tests dynamically based on too old/too new/explicitly skipped LLVM major versions. Test plan: - CI - Used in stacked PRs --- oi/OICompiler.cpp | 4 ++++ oi/OICompiler.h | 5 +++++ test/integration/CMakeLists.txt | 1 + test/integration/gen_tests.py | 34 ++++++++++++++++++++++++++++++ test/integration/runner_common.cpp | 2 ++ 5 files changed, 46 insertions(+) diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp index a36941cd..1832325c 100644 --- a/oi/OICompiler.cpp +++ b/oi/OICompiler.cpp @@ -738,4 +738,8 @@ std::optional OICompiler::applyRelocs( return res; } +int OICompiler::getLLVMVersionMajor() { + return LLVM_VERSION_MAJOR; +} + } // namespace oi::detail diff --git a/oi/OICompiler.h b/oi/OICompiler.h index 2ce4544f..1d59eb3a 100644 --- a/oi/OICompiler.h +++ b/oi/OICompiler.h @@ -183,6 +183,11 @@ class OICompiler { static std::optional decodeInst(const std::vector&, uintptr_t); + /** + * @return the major version of LLVM this was compiled with + */ + static int getLLVMVersionMajor(); + private: std::shared_ptr symbols; Config config; diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index aee09ad1..ab4acb28 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -59,6 +59,7 @@ target_link_libraries(integration_test_runner PRIVATE Boost::headers ${Boost_LIBRARIES} toml + oicore ) target_compile_definitions(integration_test_runner PRIVATE TARGET_EXE_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration_test_target" diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py index d10ddd78..7eed51ba 100644 --- a/test/integration/gen_tests.py +++ b/test/integration/gen_tests.py @@ -415,6 +415,13 @@ def generate_skip(case, specific): possibly_skip = "" skip_reason = case.get("skip", False) specific_skip_reason = case.get(f"{specific}_skip", False) + + # Handle LLVM version constraints + min_llvm = case.get("min_llvm_version") + max_llvm = case.get("max_llvm_version") + skip_llvm = case.get("skip_llvm_version") + + # Check if test should be skipped due to regular skip reasons if specific_skip_reason or skip_reason: possibly_skip += " if (!run_skipped_tests) {\n" possibly_skip += " GTEST_SKIP()" @@ -425,6 +432,32 @@ def generate_skip(case, specific): possibly_skip += ";\n" possibly_skip += " }\n" + # Check LLVM version constraints + if min_llvm is not None: + possibly_skip += f" if (llvm_version_major < {min_llvm}) {{\n" + possibly_skip += f' GTEST_SKIP() << "Requires LLVM {min_llvm} or newer, but running with LLVM " << llvm_version_major;\n' + possibly_skip += " }\n" + + if max_llvm is not None: + possibly_skip += f" if (llvm_version_major > {max_llvm}) {{\n" + possibly_skip += f' GTEST_SKIP() << "Requires LLVM {max_llvm} or older, but running with LLVM " << llvm_version_major;\n' + possibly_skip += " }\n" + + if skip_llvm is not None: + if isinstance(skip_llvm, list): + for version in skip_llvm: + possibly_skip += f" if (llvm_version_major == {version}) {{\n" + possibly_skip += ( + f' GTEST_SKIP() << "Test skipped for LLVM {version}";\n' + ) + possibly_skip += " }\n" + else: + possibly_skip += f" if (llvm_version_major == {skip_llvm}) {{\n" + possibly_skip += ( + f' GTEST_SKIP() << "Test skipped for LLVM {skip_llvm}";\n' + ) + possibly_skip += " }\n" + return possibly_skip @@ -448,6 +481,7 @@ def gen_runner(output_runner_name, test_configs): "using ::testing::MatchesRegex;\n" "\n" "extern bool run_skipped_tests;\n" + "extern int llvm_version_major;\n" ) for config in test_configs: add_tests(f, config) diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp index 3c8525a3..363cbac1 100644 --- a/test/integration/runner_common.cpp +++ b/test/integration/runner_common.cpp @@ -12,6 +12,7 @@ #include #include +#include "oi/OICompiler.h" #include "oi/OIOpts.h" #include "oi/support/Toml.h" @@ -22,6 +23,7 @@ namespace bpt = boost::property_tree; namespace fs = std::filesystem; bool run_skipped_tests = false; +int llvm_version_major = oi::detail::OICompiler::getLLVMVersionMajor(); namespace { From 10bd56d136289f2387a45768519512094182ddbd Mon Sep 17 00:00:00 2001 From: Jake Hillion Date: Mon, 4 Aug 2025 14:21:43 +0100 Subject: [PATCH 2/2] nix/ci: add and enable llvm-17 --- .github/workflows/ci.yml | 4 ++-- CMakeLists.txt | 6 +++--- flake.nix | 4 +++- oi/CMakeLists.txt | 4 ++-- test/CMakeLists.txt | 4 ++-- test/integration/gen_tests.py | 3 +++ test/integration/primitives.toml | 9 ++++++++- test/integration/std_variant.toml | 6 ++++++ 8 files changed, 29 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5e07e4e..49c6c12a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,13 +24,13 @@ jobs: nix flake show --json \ | jq -r '.packages."x86_64-linux" | keys[]' \ | sed 's/^/.#/' \ - | xargs nix build + | xargs nix build --print-build-logs build-test: runs-on: 16-core-ubuntu strategy: matrix: - llvm_version: [16] + llvm_version: [16, 17] steps: - uses: actions/checkout@v4.1.7 - uses: cachix/install-nix-action@v27 diff --git a/CMakeLists.txt b/CMakeLists.txt index fa2b5258..4413212f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,8 +224,8 @@ find_package(LLVM ${LLVM_REQUESTED_VERSION} REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") -if((${LLVM_VERSION_MAJOR} VERSION_LESS 16) OR (${LLVM_VERSION_MAJOR} VERSION_GREATER 16)) - message(SEND_ERROR "Object Introspection currently requires LLVM version 16!") +if((${LLVM_VERSION_MAJOR} VERSION_LESS 16) OR (${LLVM_VERSION_MAJOR} VERSION_GREATER 17)) + message(SEND_ERROR "Object Introspection currently requires an LLVM version between 16 and 17!") endif() find_package(Clang REQUIRED CONFIG) @@ -336,7 +336,7 @@ target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) llvm_map_components_to_libnames(llvm_libs core native mcjit x86disassembler) target_link_libraries(oicore - codegen + oicodegen ${Boost_LIBRARIES} Boost::headers diff --git a/flake.nix b/flake.nix index 54ecb442..71526227 100644 --- a/flake.nix +++ b/flake.nix @@ -21,7 +21,7 @@ flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] ( system: let - defaultLlvmVersion = 16; + defaultLlvmVersion = 17; pkgs = import nixpkgs { inherit system; }; drgnSrc = pkgs.fetchFromGitHub { @@ -116,11 +116,13 @@ default = self.packages.${system}."oid-llvm${toString defaultLlvmVersion}"; oid-llvm16 = mkOidPackage 16; + oid-llvm17 = mkOidPackage 17; }; devShells = rec { default = self.devShells.${system}."oid-llvm${toString defaultLlvmVersion}"; oid-llvm16 = mkOidDevShell 16; + oid-llvm17 = mkOidDevShell 17; }; apps.default = { diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt index 70802457..9d69092c 100644 --- a/oi/CMakeLists.txt +++ b/oi/CMakeLists.txt @@ -39,12 +39,12 @@ target_link_libraries(container_info toml ) -add_library(codegen +add_library(oicodegen CodeGen.cpp FuncGen.cpp OICodeGen.cpp ) -target_link_libraries(codegen +target_link_libraries(oicodegen container_info resources symbol_service diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ccdb6062..325d6aef 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -60,7 +60,7 @@ target_compile_definitions(test_type_graph PRIVATE TARGET_EXE_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration/integration_test_target" ) target_link_libraries(test_type_graph - codegen + oicodegen container_info type_graph @@ -79,7 +79,7 @@ target_compile_definitions(test_clang_type_parser PRIVATE BUILD_DIR="${CMAKE_BINARY_DIR}" ) target_link_libraries(test_clang_type_parser - codegen + oicodegen container_info type_graph diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py index 7eed51ba..8bfcdaf6 100644 --- a/test/integration/gen_tests.py +++ b/test/integration/gen_tests.py @@ -282,6 +282,9 @@ def get_config_strings(case): def add_oid_integration_test(f, config, case_name, case): + if "oid_disable" in case: + return + probe_type = case.get("type", "entry") args = case.get("args", "arg0") diff --git a/test/integration/primitives.toml b/test/integration/primitives.toml index 94784fb4..2a4bee6a 100644 --- a/test/integration/primitives.toml +++ b/test/integration/primitives.toml @@ -71,7 +71,14 @@ param_types = ["double"] setup = "return 3.14;" expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]' - [cases.long_double] + + [cases.oil_long_double] + oid_disable = "Split test case for OID" + param_types = ["long double"] + setup = "return 3.14;" + expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "size":16}]' + [cases.oid_long_double] + skip_llvm_version = 17 param_types = ["long double"] setup = "return 3.14;" expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "size":16}]' diff --git a/test/integration/std_variant.toml b/test/integration/std_variant.toml index c63a35c9..79c0e0eb 100644 --- a/test/integration/std_variant.toml +++ b/test/integration/std_variant.toml @@ -196,6 +196,9 @@ definitions = ''' # 0xff can be a valid index if there are at least 256 parameters, and that # the invalid index value is raised to 0xffff. [cases.256_params_256] + # These tests take 20 minutes each on LLVM 17 and fail because they can't + # link memset (at least on my machine). Annoyingly, disable for now. + skip_llvm_version = 17 param_types = ["const std::variant&"] setup = "return 'a';" expect_json = '''[{ @@ -219,6 +222,9 @@ definitions = ''' ] }]''' [cases.256_params_empty] + # These tests take 20 minutes each on LLVM 17 and fail because they can't + # link memset (at least on my machine). Annoyingly, disable for now. + skip_llvm_version = 17 param_types = ["const std::variant&"] setup = ''' std::variant var{'a'};