From 9199011134df8ba95a12b8703fe708a947f06c7f Mon Sep 17 00:00:00 2001 From: Calvin Beck Date: Tue, 12 Aug 2025 13:44:44 -0400 Subject: [PATCH] Fix nix build for c2rust. This commit adds a number of improvements to the nix build for c2rust the main highlights are: - A c2rust package is provided, allowing nix users to install and run c2rust itself, instead of just getting a development environment from nix. - Update LLVM / clang to version 18. - Test cases pass in the c2rust dev environment now. Some fixes include... - Removing oxalica and just using fenix. Mixing rust overlays in nix was causing issues. - Use the nix tinycbor package to build c2rust. The c2rust cmake files add an external dependency on tinycbor, which causes sandboxed nix builds to fail because they cannot fetch the package. This involves patching the CMakeLists.txt file, and using pkg-config to find the system tinycbor library. Note: in the dev shell cmake will still fetch the external tinycbor. - Use the C2RUST_TINYCBOR_PATH environment variable, if it exists, to find tinycbor in c2rust-ast-exporter's build.rs - Use bindgenHook to make sure Rust's bindgen can find header files. - Update the test_translator.py script to include nix flags in compiler_commands.json Messing with bindgenHook, disable checks so package builds. - test_translator.py will use nix dependencies when the environment variable C2RUST_USE_NIX=1 + Passing the --use-nix flag to test_translator.py will also explicitly force the script to use nix dependencies. --- c2rust-ast-exporter/build.rs | 12 +++- flake.lock | 119 ++++++----------------------------- flake.nix | 117 ++++++++++++++++++++++++---------- nix-tinycbor-cmake.patch | 76 ++++++++++++++++++++++ scripts/test_translator.py | 31 +++++++-- 5 files changed, 215 insertions(+), 140 deletions(-) create mode 100644 nix-tinycbor-cmake.patch diff --git a/c2rust-ast-exporter/build.rs b/c2rust-ast-exporter/build.rs index 968e3d6d5a..b174221e59 100644 --- a/c2rust-ast-exporter/build.rs +++ b/c2rust-ast-exporter/build.rs @@ -149,8 +149,18 @@ fn build_native(llvm_info: &LLVMInfo) { } }; - // Statically link against 'clangAstExporter' which requires 'tinycbor' + // Use a custom tinybor directory via an environment variable + if let Ok(tinycbor_dir) = env::var("TINYCBOR_DIR") { + let include_dir = Path::new(&tinycbor_dir).join("include"); + let lib_dir = Path::new(&tinycbor_dir).join("lib"); + + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + println!("cargo:rerun-if-changed={}", include_dir.display()); + } + println!("cargo:rustc-link-lib=static=tinycbor"); + + // Statically link against 'clangAstExporter' which requires 'tinycbor' println!("cargo:rustc-link-lib=static=clangAstExporter"); println!("cargo:rustc-link-search=native={}", llvm_lib_dir); diff --git a/flake.lock b/flake.lock index e8f1936223..bddddd9f84 100644 --- a/flake.lock +++ b/flake.lock @@ -2,15 +2,17 @@ "nodes": { "fenix": { "inputs": { - "nixpkgs": "nixpkgs", + "nixpkgs": [ + "nixpkgs" + ], "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1715581585, - "narHash": "sha256-/JjvIn1NXW3yOaDcD8Me987/QcXjo+rhg+uThasPAnI=", + "lastModified": 1754980813, + "narHash": "sha256-Wr9ei2V4rfr3HR5eJUA7pjMIrHH5o4DtWazQC5UwxHA=", "owner": "nix-community", "repo": "fenix", - "rev": "2c4905096782e8e908205e7fa54ef987fba62793", + "rev": "a1ce805b08279ee4e697b47aa3aa28fe2b335de6", "type": "github" }, "original": { @@ -19,107 +21,37 @@ "type": "github" } }, - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1705309234, - "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, "nixpkgs": { "locked": { - "lastModified": 1715534503, - "narHash": "sha256-5ZSVkFadZbFP1THataCaSf0JH2cAH3S29hU9rrxTEqk=", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "2057814051972fa1453ddfb0d98badbea9b83c06", - "type": "github" - }, - "original": { - "owner": "nixos", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { - "lastModified": 1715542476, - "narHash": "sha256-FF593AtlzQqa8JpzrXyRws4CeKbc5W86o8tHt4nRfIg=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "44072e24566c5bcc0b7aa9178a0104f4cfffab19", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-23.11", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_3": { - "locked": { - "lastModified": 1706487304, - "narHash": "sha256-LE8lVX28MV2jWJsidW13D2qrHU/RUUONendL2Q/WlJg=", + "lastModified": 1754725699, + "narHash": "sha256-iAcj9T/Y+3DBy2J0N+yF9XQQQ8IEb5swLFzs23CdP88=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "90f456026d284c22b3e3497be980b2e47d0b28ac", + "rev": "85dbfc7aaf52ecb755f87e577ddbe6dbbdbc1054", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixpkgs-unstable", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } }, - "oxalica": { - "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs_3" - }, - "locked": { - "lastModified": 1715652909, - "narHash": "sha256-aCLEDvzL1j51Rf2mCFOqK1mieMO3pAn5ItCIdr5h2LA=", - "owner": "oxalica", - "repo": "rust-overlay", - "rev": "1d8fcbbfcfd3476c2665384a46ee9d07ef2b4dd9", - "type": "github" - }, - "original": { - "owner": "oxalica", - "repo": "rust-overlay", - "type": "github" - } - }, "root": { "inputs": { "fenix": "fenix", - "nixpkgs": "nixpkgs_2", - "oxalica": "oxalica", + "nixpkgs": "nixpkgs", "utils": "utils" } }, "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1715255944, - "narHash": "sha256-vLLgYpdtKBaGYTamNLg1rbRo1bPXp4Jgded/gnprPVw=", + "lastModified": 1754926538, + "narHash": "sha256-fuHLsvM5z5/5ia3yL0/mr472wXnxSrtXECa+pspQchA=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "5bf2f85c8054d80424899fa581db1b192230efb5", + "rev": "9db05508ed08a4c952017769b45b57c4ad505872", "type": "github" }, "original": { @@ -144,31 +76,16 @@ "type": "github" } }, - "systems_2": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, "utils": { "inputs": { - "systems": "systems_2" + "systems": "systems" }, "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index f78bdecb70..caff716de2 100644 --- a/flake.nix +++ b/flake.nix @@ -2,54 +2,69 @@ description = "Flake for c2rust"; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; utils.url = "github:numtide/flake-utils"; fenix = { url = "github:nix-community/fenix"; - }; - oxalica = { - url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; }; }; - outputs = inputs@{ self, nixpkgs, utils, fenix, oxalica}: + outputs = inputs@{ self, nixpkgs, utils, fenix}: utils.lib.eachDefaultSystem (system: - let - fenixStable = fenix.packages.${system}.fromToolchainFile { - file = ./rust-toolchain.toml; - sha256 = "sha256-r/8YBFuFa4hpwgE3FnME7nQA2Uc1uqj0eCE1NWmI1u0"; - }; + let + fenixToolchain = + let + toml = with builtins; (fromTOML (readFile ./rust-toolchain.toml)).toolchain; + in + (fenix.packages.${system}.fromToolchainName { + name = toml.channel; + sha256 = "sha256-r/8YBFuFa4hpwgE3FnME7nQA2Uc1uqj0eCE1NWmI1u0"; + })."completeToolchain"; + pkgs = import nixpkgs { inherit system; - overlays = [ - (import oxalica) - ]; + overlays = [ ]; config = { allowUnfree = true; }; }; - in { - defaultPackage = self.devShell.${system}; - devShell = pkgs.mkShell.override { } { - LIBCLANG_PATH = "${pkgs.llvmPackages_14.libclang.lib}/lib"; - CMAKE_LLVM_DIR = "${pkgs.llvmPackages_14.libllvm.dev}/lib/cmake/llvm"; - CMAKE_CLANG_DIR = "${pkgs.llvmPackages_14.libclang.dev}/lib/cmake/clang"; - shellHook = '' - export CARGO_TARGET_DIR="$(git rev-parse --show-toplevel)/target_dirs/nix_rustc"; - ''; - RUST_SRC_PATH = pkgs.rustPlatform.rustLibSrc; - buildInputs = + myLLVM = pkgs.llvmPackages_18; + myStdenv = pkgs.clang18Stdenv; + + rustPlatform = pkgs.makeRustPlatform { + cargo = fenixToolchain; + rustc = fenixToolchain; + }; + env = with pkgs; { + LIBCLANG_PATH = "${myLLVM.libclang.lib}/lib"; + CMAKE_LLVM_DIR = "${myLLVM.libllvm.dev}/lib/cmake/llvm"; + CMAKE_CLANG_DIR = "${myLLVM.libclang.dev}/lib/cmake/clang"; + LLVM_CONFIG_PATH = "${myLLVM.libllvm.dev}/bin/llvm-config"; + CLANG_PATH = "${myLLVM.clang}/bin/clang"; + TINYCBOR_DIR = "${pkgs.tinycbor}"; + NIX_ENFORCE_NO_NATIVE = 0; # Enable SSE instructions. + # Enable nix in the c2rust test suite + # This flag is used to tell the test scripts to look for + # libraries under nix paths. + C2RUST_USE_NIX = 1; + RUST_SRC_PATH = "${fenixToolchain}/lib/rustlib/src/rust/library"; + }; + in rec { + packages = { + default = rustPlatform.buildRustPackage (with pkgs; env // { + pname = "c2rust"; + version = "0.20.0"; + src = ./.; + doCheck = false; # Can use checkFlags to disable specific tests + + patches = [ ./nix-tinycbor-cmake.patch ]; + + nativeBuildInputs = with pkgs; [ - clangStdenv.cc - llvmPackages_14.libclang pkg-config - fenix.packages.${system}.rust-analyzer - llvmPackages_14.clang cmake - llvmPackages_14.llvm - llvmPackages_14.libllvm - openssl (python3.withPackages (python-pkgs: with python-pkgs; @@ -68,9 +83,43 @@ ] ) ) + myStdenv.cc + myLLVM.libclang + myLLVM.clang + myLLVM.llvm + myLLVM.libllvm + ]; + + buildInputs = + with pkgs; [ + rustPlatform.bindgenHook + myStdenv.cc + myLLVM.libclang + myLLVM.clang + myLLVM.llvm + myLLVM.libllvm + tinycbor + openssl zlib - (rust-bin.fromRustupToolchainFile ./rust-toolchain.toml) + fenixToolchain ]; - }; - }); + + cargoLock = { + lockFile = ./Cargo.lock; + }; + }); + }; + defaultPackage = packages.default; + + devShells = { + # Include a fixed version of clang in the development environment for testing. + default = pkgs.mkShell (with pkgs; env // { + strictDeps = true; + inputsFrom = [ packages.default ]; + buildInputs = [ ]; + }); + }; + + devShell = devShells.default; + }); } diff --git a/nix-tinycbor-cmake.patch b/nix-tinycbor-cmake.patch new file mode 100644 index 0000000000..e6ddee5a77 --- /dev/null +++ b/nix-tinycbor-cmake.patch @@ -0,0 +1,76 @@ +--- a/c2rust-ast-exporter/src/CMakeLists.txt 2025-08-22 16:55:32.191186099 -0400 ++++ b/c2rust-ast-exporter/src/CMakeLists.txt 2025-08-22 16:56:04.157045169 -0400 +@@ -1,6 +1,7 @@ + cmake_minimum_required(VERSION 3.5...4.0) + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + project(ASTExporter) ++set(CMAKE_VERBOSE_MAKEFILE ON) + + ################################################# + # TinyCBOR # +@@ -21,28 +22,12 @@ + set(MAKE "make") + endif() + +-include(ExternalProject) +-ExternalProject_Add(tinycbor_build +- PREFIX ${TINYCBOR_PREFIX} +- INSTALL_DIR ${CMAKE_BINARY_DIR} +- GIT_REPOSITORY ${TINYCBOR_REPO} +- GIT_TAG ${TINYCBOR_TAG} +- # the fd redirection here fails when the build run inside Cargo. +- # patch from upstream: +- # https://github.com/intel/tinycbor/commit/6176e0a28d7c5ef3a5e9cbd02521999c412de72c +- PATCH_COMMAND patch --forward -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/tinycbor_fix_build.patch || true +- CONFIGURE_COMMAND ${MAKE} .config && cat ${CMAKE_CURRENT_SOURCE_DIR}/tinycbor.config >> .config +- BUILD_COMMAND ${MAKE} --quiet prefix= CFLAGS=-fPIC +- INSTALL_COMMAND ${MAKE} --quiet prefix= install +- BUILD_IN_SOURCE 1 +- BUILD_BYPRODUCTS ${CMAKE_BINARY_DIR}/lib/libtinycbor.a +-) +- + include_directories(${CMAKE_BINARY_DIR}/include) + +-add_library(tinycbor STATIC IMPORTED) +-set_target_properties(tinycbor PROPERTIES IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/lib/libtinycbor.a) +-add_dependencies(tinycbor tinycbor_build) ++find_package(PkgConfig REQUIRED) ++pkg_check_modules(TINYCBOR REQUIRED tinycbor) ++include_directories(${TINYCBOR_INCLUDE_DIRS}) ++link_directories(${TINYCBOR_LIBRARY_DIRS}) + + set(AST_EXPORTER_SRCS + AstExporter.cpp +@@ -93,6 +78,9 @@ + CXX_STANDARD 17 + CXX_EXTENSIONS OFF + ) ++ ++target_link_directories(c2rust-ast-exporter PRIVATE ${TINYCBOR_LIBRARY_DIRS}) ++ + # PRIVATE was added to make c2rust-ast-exporter build with LLVM 6.0. Keyword + # description: https://cmake.org/pipermail/cmake/2016-May/063400.html + target_link_libraries(c2rust-ast-exporter PRIVATE +@@ -101,18 +89,20 @@ + clangTooling + clangBasic + clangASTMatchers +- tinycbor ++ ${TINYCBOR_LIBRARIES} + ) + + set_target_properties(clangAstExporter PROPERTIES + CXX_STANDARD 17 # will decay to 14 if compiler doesn't support c++17 + CXX_EXTENSIONS OFF + ) ++ ++target_link_directories(clangAstExporter PRIVATE ${TINYCBOR_LIBRARY_DIRS}) + target_link_libraries(clangAstExporter PRIVATE + clangAST + clangFrontend + clangTooling + clangBasic + clangASTMatchers +- tinycbor ++ ${TINYCBOR_LIBRARIES} + ) diff --git a/scripts/test_translator.py b/scripts/test_translator.py index 6ad2ff86e7..11f9d92013 100755 --- a/scripts/test_translator.py +++ b/scripts/test_translator.py @@ -46,6 +46,9 @@ 'cc_db', 'c_obj', 'c_lib', 'rust_src', ] +# Whether or not to use nix for c2rust dependencies. This adds paths to +# nix libraries in the generated compile_commands.json files. +C2RUST_USE_NIX = False class TestOutcome(Enum): Success = "successes" @@ -244,11 +247,22 @@ def __init__(self, full_path: str, files: 're.Pattern', keep: List[str], log_lev # are broken without it on macOS 12. # limit this to macOS because if we do happen to have multiple versions of Clang around, we # don't know which to use here, and using the wrong can one break things badly - if sys.platform == "darwin": + if sys.platform == "darwin" or C2RUST_USE_NIX: _, stdout, _ = clang["-print-resource-dir"].run(retcode=None) - self.clang_resource_dir = " \"-I{}/include\",".format(stdout.strip()) + clang_resource_dir = " \"-I{}/include\",".format(stdout.strip()) else: - self.clang_resource_dir = "" + clang_resource_dir = "" + + self.clang_extra_flags = clang_resource_dir + + # The compile_commands.json needs to have extra flags to tell + # it where to find libraries when using nix. + if C2RUST_USE_NIX: + nix_compile_flags = os.environ.get("BINDGEN_EXTRA_CLANG_ARGS", "") + os.environ.get("NIX_CFLAGS_COMPILE", "") + nix_compile_flags = nix_compile_flags.replace("-isystem ", "-I").replace("-idirafter ", "-I") + nix_compile_flags = ['"{}"'.format(flag) for flag in nix_compile_flags.split() if flag.startswith("-I")] + nix_compile_flags = ", ".join(nix_compile_flags) + "," + self.clang_extra_flags += nix_compile_flags # parse target arch from directory name if it includes a dot split_by_dots = self.name.split('.') @@ -340,7 +354,7 @@ def _generate_cc_db(self, c_file_path: str) -> None: "file": "{0}" }} ] - """.format(cfile, directory, target_args, self.clang_resource_dir) + """.format(cfile, directory, target_args, self.clang_extra_flags) cc_db = os.path.join(directory, "compile_commands.json") @@ -598,6 +612,7 @@ def get_testdirectories( def main() -> None: + global C2RUST_USE_NIX desc = 'run regression / unit / feature tests.' parser = argparse.ArgumentParser(description=desc) parser.add_argument('directory', type=readable_directory) @@ -619,6 +634,11 @@ def main() -> None: choices=intermediate_files + ['all'], default=[], help="Which intermediate files to not clear" ) + parser.add_argument( + '--use-nix', dest='use_nix', action='store_true', + default=os.environ.get('C2RUST_USE_NIX') == '1', + help="Use nix dependencies for c2rust. By default this is set if the C2RUST_USE_NIX environment variable is set to 1." + ) c.add_args(parser) args = parser.parse_args() @@ -631,6 +651,9 @@ def main() -> None: logging.debug("args: %s", " ".join(sys.argv)) + # Set whether we are using nix. + C2RUST_USE_NIX=args.use_nix + # check that the binaries have been built first bins = [c.TRANSPILER] for b in bins: