diff --git a/MODULE.bazel b/MODULE.bazel index 97d50e9e835e5..0712935dd5163 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -118,6 +118,9 @@ http_archive( "@carbon//bazel/llvm_project:0002_Added_Bazel_build_for_compiler_rt_fuzzer.patch", "@carbon//bazel/llvm_project:0003_Comment_out_unloaded_proto_library_dependencies.patch", "@carbon//bazel/llvm_project:0004_Introduce_basic_sources_exporting_for_libunwind.patch", + "@carbon//bazel/llvm_project:0005_Introduce_basic_sources_exporting_for_libcxx_and_libcxxabi.patch", + "@carbon//bazel/llvm_project:0006_Add_a_filegroup_containing__all__sources_to_the_libc_build_rules.patch", + "@carbon//bazel/llvm_project:0007_Remove_target_compatibility_restrictions_for_float128.patch", ], strip_prefix = "llvm-project-{0}".format(llvm_project_version), urls = ["https://github.com/llvm/llvm-project/archive/{0}.tar.gz".format(llvm_project_version)], diff --git a/bazel/check_deps/check_non_test_cc_deps.py b/bazel/check_deps/check_non_test_cc_deps.py index 9bc6b66125b95..c7604a64974c4 100644 --- a/bazel/check_deps/check_non_test_cc_deps.py +++ b/bazel/check_deps/check_non_test_cc_deps.py @@ -41,11 +41,14 @@ # Other packages in the LLVM project shouldn't be accidentally used # in Carbon. We can expand the above list if use cases emerge. if package not in ( - "llvm", - "lld", "clang", "clang-tools-extra/clangd", + "libc", + "libcxx", + "libcxxabi", "libunwind", + "lld", + "llvm", # While this is in a `third_party` directory, its code is documented # as part of LLVM and for use in compiler-rt. "third-party/siphash", diff --git a/bazel/llvm_project/0005_Introduce_basic_sources_exporting_for_libcxx_and_libcxxabi.patch b/bazel/llvm_project/0005_Introduce_basic_sources_exporting_for_libcxx_and_libcxxabi.patch new file mode 100644 index 0000000000000..97ee577c5c093 --- /dev/null +++ b/bazel/llvm_project/0005_Introduce_basic_sources_exporting_for_libcxx_and_libcxxabi.patch @@ -0,0 +1,96 @@ +Commit ID: e4ff7299fe7e35e70ba79f5d8e2c58658cfba678 +Change ID: mstnwoqruyypnoouksnyqssllrsozpos +Bookmarks: bz-libcxx bz-libcxx@git bz-libcxx@origin +Author : Chandler Carruth (2025-09-25 22:55:26) +Committer: Chandler Carruth (2025-11-22 09:23:52) + + Introduce basic sources exporting for libcxx and libcxxabi + + This exports the source files directly so that they can be used to build + a libcxx runtime library on demand. + +diff --git a/utils/bazel/llvm-project-overlay/libcxx/BUILD.bazel b/utils/bazel/llvm-project-overlay/libcxx/BUILD.bazel +new file mode 100644 +index 0000000000..a81a64c649 +--- /dev/null ++++ b/utils/bazel/llvm-project-overlay/libcxx/BUILD.bazel +@@ -0,0 +1,49 @@ ++# This file is licensed under the Apache License v2.0 with LLVM Exceptions. ++# See https://llvm.org/LICENSE.txt for license information. ++# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++ ++licenses(["notice"]) ++ ++package( ++ default_visibility = ["//visibility:public"], ++) ++ ++exports_files(["include/__config_site.in"]) ++ ++exports_files(["vendor/llvm/default_assertion_handler.in"]) ++ ++filegroup( ++ name = "libcxx_hdrs", ++ srcs = glob( ++ [ ++ # Top level includes and those in `experimental` and `ext` sometimes ++ # have no extension. ++ "include/*", ++ "include/experimental/*", ++ "include/ext/*", ++ ++ # Implementation detail headers all use `.h` extensions ++ "include/**/*.h", ++ ], ++ exclude = [ ++ # Omit CMake and CMake-configured files that get caught by the ++ # extension-less patterns. ++ "**/*.in", ++ "**/CMakeLists.txt", ++ ++ # Omit C++03 compatibility headers as current users don't need them. ++ "include/__cxx03/**", ++ ], ++ ), ++) ++ ++filegroup( ++ name = "libcxx_srcs", ++ srcs = glob( ++ [ ++ "src/**/*.cpp", ++ "src/**/*.h", ++ "src/**/*.ipp", ++ ], ++ ), ++) +diff --git a/utils/bazel/llvm-project-overlay/libcxxabi/BUILD.bazel b/utils/bazel/llvm-project-overlay/libcxxabi/BUILD.bazel +new file mode 100644 +index 0000000000..cf491e4e46 +--- /dev/null ++++ b/utils/bazel/llvm-project-overlay/libcxxabi/BUILD.bazel +@@ -0,0 +1,24 @@ ++# This file is licensed under the Apache License v2.0 with LLVM Exceptions. ++# See https://llvm.org/LICENSE.txt for license information. ++# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++ ++licenses(["notice"]) ++ ++package( ++ default_visibility = ["//visibility:public"], ++) ++ ++filegroup( ++ name = "libcxxabi_hdrs", ++ srcs = glob(["include/*.h"]), ++) ++ ++filegroup( ++ name = "libcxxabi_srcs", ++ srcs = glob([ ++ "src/**/*.cpp", ++ "src/**/*.def", ++ "src/**/*.inc", ++ "src/**/*.h", ++ ]), ++) diff --git a/bazel/llvm_project/0006_Add_a_filegroup_containing__all__sources_to_the_libc_build_rules.patch b/bazel/llvm_project/0006_Add_a_filegroup_containing__all__sources_to_the_libc_build_rules.patch new file mode 100644 index 0000000000000..5fd61dd403f1b --- /dev/null +++ b/bazel/llvm_project/0006_Add_a_filegroup_containing__all__sources_to_the_libc_build_rules.patch @@ -0,0 +1,31 @@ +Commit ID: e524424ed32e945bdced8c19ca480f461d04397a +Change ID: lslousqwyovpkqlxvyrzmmypxkxxulwn +Bookmarks: push-lslousqwyovp push-lslousqwyovp@git push-lslousqwyovp@origin +Author : Chandler Carruth (2025-11-20 03:05:36) +Committer: Chandler Carruth (2025-11-22 09:23:52) + + Add a filegroup containing _all_ sources to the libc build rules + + These rules already expose a filegroup containing the _dependencies_, + but that misses the source files directly in the top level library. + + Without this filegroup, there isn't a way to access the source files + used by libcxx when building it, etc. + +diff --git a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl +index 0f2965369c..c871334dd9 100644 +--- a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl ++++ b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl +@@ -247,6 +247,12 @@ + **kwargs + ) + ++ _libc_srcs_filegroup( ++ name = name + "_hdrs", ++ libs = [":" + name], ++ enforce_headers_only = True, ++ ) ++ + def libc_generated_header(name, hdr, yaml_template, other_srcs = []): + """Generates a libc header file from YAML template. + diff --git a/bazel/llvm_project/0007_Remove_target_compatibility_restrictions_for_float128.patch b/bazel/llvm_project/0007_Remove_target_compatibility_restrictions_for_float128.patch new file mode 100644 index 0000000000000..6938f9c4d3e97 --- /dev/null +++ b/bazel/llvm_project/0007_Remove_target_compatibility_restrictions_for_float128.patch @@ -0,0 +1,36 @@ +Commit ID: 94325bae07213d2dd8263945f5362720dee7bcb9 +Change ID: kskkxwvnyqwuzkswkuxpnmyovqxoypnz +Bookmarks: push-kskkxwvnyqwu push-kskkxwvnyqwu@git push-kskkxwvnyqwu@origin +Author : Chandler Carruth (2025-11-24 07:26:06) +Committer: Chandler Carruth (2025-11-24 07:40:58) + + Remove target compatibility restrictions for float128 + + The restrictions here aren't nearly as much about the OS as the compiler + and architecture, but the Bazel restriction was OS-based. Everything + seems to work well on even Arm64 macOS, and I would expect most BSDs and + other OSes to work well with Clang's support on x86-64. + + The source code here already handles detecting when there is compiler + support for the type. And the users of this don't `select` or do + anything else to conditionally include the header, so it seems better to + not restrict access to the header from the build system, and instead + continue making the source code compatible or a no-op on relevant + configurations. + +diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +index bd48222856..e3962d9b5f 100644 +--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel ++++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +@@ -171,11 +171,6 @@ + libc_support_library( + name = "llvm_libc_types_float128", + hdrs = ["include/llvm-libc-types/float128.h"], +- target_compatible_with = select({ +- "@platforms//os:linux": [], +- "@platforms//os:windows": [], +- "//conditions:default": ["@platforms//:incompatible"], +- }), + deps = [":llvm_libc_macros_float_macros"], + ) + diff --git a/toolchain/base/runtime_sources.bzl b/toolchain/base/runtime_sources.bzl index 1fce76f345f8d..c0098f5836d6d 100644 --- a/toolchain/base/runtime_sources.bzl +++ b/toolchain/base/runtime_sources.bzl @@ -38,7 +38,9 @@ BUILTINS_FILEGROUPS = { } RUNTIMES_FILEGROUPS = { - "libunwind_srcs": "@llvm-project//libunwind:libunwind_srcs", + "libcxx": "@llvm-project//libcxx:libcxx_srcs", + "libcxxabi": "@llvm-project//libcxxabi:libcxxabi_srcs", + "libunwind": "@llvm-project//libunwind:libunwind_srcs", } _TEMPLATE = """ @@ -89,8 +91,16 @@ inline constexpr llvm::StringLiteral BuiltinsI386Srcs[] = {{ {i386_srcs} }}; +constexpr inline llvm::StringLiteral LibcxxSrcs[] = {{ +{libcxx} +}}; + +constexpr inline llvm::StringLiteral LibcxxabiSrcs[] = {{ +{libcxxabi} +}}; + constexpr inline llvm::StringLiteral LibunwindSrcs[] = {{ -{libunwind_srcs} +{libunwind} }}; }} // namespace Carbon::RuntimeSources @@ -118,14 +128,14 @@ def _get_path(file_attr, to_path_fn): return '"{0}"'.format(to_path_fn(files[0])) -def _get_paths(files_attr, to_path_fn): +def _get_paths(files_attr, to_path_fn, prefix = ""): files = [] for src in files_attr: files.extend(src[DefaultInfo].files.to_list()) files.extend(src[DefaultInfo].default_runfiles.files.to_list()) return "\n".join([ - ' "{0}",'.format(to_path_fn(f)) + ' "{0}{1}",'.format(prefix, to_path_fn(f)) for f in files ]) @@ -138,7 +148,9 @@ def _generate_runtime_sources_h_rule(ctx): k: _get_paths(getattr(ctx.attr, "_" + k), _builtins_path) for k in BUILTINS_FILEGROUPS.keys() } | { - k: _get_paths(getattr(ctx.attr, "_" + k), _runtimes_path) + # Other runtimes are installed under separate directories named the same + # as their key. + k: _get_paths(getattr(ctx.attr, "_" + k), _runtimes_path, k + "/") for k in RUNTIMES_FILEGROUPS.keys() }))) return [DefaultInfo(files = depset([h_file]))] diff --git a/toolchain/driver/BUILD b/toolchain/driver/BUILD index c2624c2ba2c91..d0eed97c182a6 100644 --- a/toolchain/driver/BUILD +++ b/toolchain/driver/BUILD @@ -82,7 +82,7 @@ cc_test( cc_test( name = "clang_runtimes_test", - size = "small", + size = "medium", srcs = ["clang_runtimes_test.cpp"], data = ["//toolchain/install:install_data"], deps = [ diff --git a/toolchain/driver/build_runtimes_subcommand.cpp b/toolchain/driver/build_runtimes_subcommand.cpp index e6a9dc514604c..a00831a2347cc 100644 --- a/toolchain/driver/build_runtimes_subcommand.cpp +++ b/toolchain/driver/build_runtimes_subcommand.cpp @@ -97,9 +97,13 @@ auto BuildRuntimesSubcommand::RunInternal(DriverEnv& driver_env) ClangArchiveRuntimesBuilder lib_unwind_builder( &runner, driver_env.thread_pool, llvm::Triple(features.target), &runtimes); + ClangArchiveRuntimesBuilder libcxx_builder( + &runner, driver_env.thread_pool, llvm::Triple(features.target), + &runtimes); CARBON_RETURN_IF_ERROR(std::move(resource_dir_builder).Wait()); CARBON_RETURN_IF_ERROR(std::move(lib_unwind_builder).Wait()); + CARBON_RETURN_IF_ERROR(std::move(libcxx_builder).Wait()); return runtimes.base_path(); } diff --git a/toolchain/driver/clang_runtimes.cpp b/toolchain/driver/clang_runtimes.cpp index 75169ad391a3f..6bab3c1c6c2f2 100644 --- a/toolchain/driver/clang_runtimes.cpp +++ b/toolchain/driver/clang_runtimes.cpp @@ -8,7 +8,9 @@ #include #include +#include #include +#include #include #include #include @@ -47,7 +49,7 @@ auto ClangRuntimesBuilderBase::ArchiveBuilder::Setup(Latch::Handle latch_handle) // manually populate the vector with errors that we'll replace with the actual // result in each thread. objs_.reserve(src_files_.size()); - for (auto _ : src_files_) { + for (const auto& _ : src_files_) { objs_.push_back(Error("Never constructed archive member!")); } @@ -163,20 +165,26 @@ auto ClangRuntimesBuilderBase::ArchiveBuilder::CompileMember( llvm::StringRef src_file) -> ErrorOr { // Create any obj subdirectories needed for this file. CARBON_RETURN_IF_ERROR(CreateObjDir(src_file.str())); - + std::filesystem::path src_path = srcs_root_ / std::string_view(src_file); std::filesystem::path obj_path = builder_->runtimes_builder_->path() / std::string_view(src_file); obj_path += ".o"; - std::filesystem::path src_path = srcs_path_ / std::string_view(src_file); - CARBON_VLOG("Building `{0}' from `{1}`...\n", obj_path, src_path); + CARBON_VLOG("Building `{0}' from `{1}`...\n", obj_path, src_file); llvm::SmallVector args(cflags_); // Add language-specific flags based on file extension. + // + // Currently, we hard code a sufficiently "recent" C++ standard, but this is + // arbitrary and brittle. We'll have to update these any time one of the + // libraries uses a too-new feature. + // + // TODO: We should eventually switch to something more like `/std:c++latest` + // in MSVC-style command lines, but would need that implemented in Clang. if (src_file.ends_with(".c")) { args.push_back("-std=c11"); } else if (src_file.ends_with(".cpp")) { - args.push_back("-std=c++20"); + args.push_back("-std=c++26"); } // Collect the additional required flags and dynamic flags for this builder. @@ -214,7 +222,7 @@ auto ClangRuntimesBuilderBase::ArchiveBuilder::CompileMember( } template - requires(Component == Runtimes::LibUnwind) + requires IsClangArchiveRuntimes ClangArchiveRuntimesBuilder::ClangArchiveRuntimesBuilder( ClangRunner* clang, llvm::ThreadPoolInterface* threads, llvm::Triple target_triple, Runtimes* runtimes) @@ -246,29 +254,87 @@ ClangArchiveRuntimesBuilder::ClangArchiveRuntimesBuilder( } if constexpr (Component == Runtimes::LibUnwind) { - srcs_path_ = installation().libunwind_path(); - include_path_ = installation().libunwind_path() / "include"; archive_path_ = std::filesystem::path("lib") / "libunwind.a"; + include_paths_ = {installation().libunwind_path() / "include"}; + } else if constexpr (Component == Runtimes::Libcxx) { + archive_path_ = std::filesystem::path("lib") / "libc++.a"; + include_paths_ = { + installation().libcxx_path() / "include", + // Some private headers of libc++ are nested in the source directory. + installation().libcxx_path() / "src", + installation().libcxxabi_path() / "include", + // Libc++ also uses llvm-libc header-only libraries for parts of its + // implementation. All the `#include`s are relative to the root of the + // internal libc source tree rather than an `include` directory. + installation().libc_path() / "internal", + }; } else { static_assert(false, "Invalid runtimes component for an archive runtime builder."); } - archive_.emplace(this, archive_path_, srcs_path_, CollectSrcFiles(), - CollectCflags()); + archive_.emplace(this, archive_path_, installation().runtimes_root(), + CollectSrcFiles(), CollectCflags()); tasks_.async([this]() mutable { Setup(); }); } template - requires(Component == Runtimes::LibUnwind) + requires IsClangArchiveRuntimes auto ClangArchiveRuntimesBuilder::CollectSrcFiles() -> llvm::SmallVector { if constexpr (Component == Runtimes::LibUnwind) { - return llvm::SmallVector(llvm::make_filter_range( + return llvm::to_vector_of(llvm::make_filter_range( RuntimeSources::LibunwindSrcs, [](llvm::StringRef src) { return src.ends_with(".c") || src.ends_with(".cpp") || src.ends_with(".S"); })); + } else if constexpr (Component == Runtimes::Libcxx) { + auto libcxx_srcs = llvm::make_filter_range( + RuntimeSources::LibcxxSrcs, [this](llvm::StringRef src) { + if (!src.ends_with(".cpp")) { + return false; + } + + // We include libc++abi and so don't need new/delete definitions. + if (src == "libcxx/src/new.cpp") { + return false; + } + // We use compiler-rt for builtins, so we don't need int128 helpers. + if (src == "libcxx/src/filesystem/int128_builtins.cpp") { + return false; + } + + // We don't currently use the libdispatch PSTL backend. + // TODO: We should evaluate enabling this on macOS. + if (src == "libcxx/src/pstl/libdispatch.cpp") { + return false; + } + + // Skip platform-specific code for unsupported platforms. + // TODO: We should revisit this and include the code for these targets + // along with testing to make sure it works. + if (src.starts_with("libcxx/src/support/ibm/") || + src.starts_with("libcxx/src/support/win32/")) { + return false; + } + + // The timezone database is currently only enabled on Linux in + // upstream. + if (!target_triple_.isOSLinux() && + (src == "libcxx/src/experimental/chrono_exception.cpp" || + src == "libcxx/src/experimental/time_zone.cpp" || + src == "libcxx/src/experimental/tzdb.cpp" || + src == "libcxx/src/experimental/tzdb_list.cpp")) { + return false; + } + + return true; + }); + auto libcxxabi_srcs = llvm::make_filter_range( + RuntimeSources::LibcxxabiSrcs, + [](llvm::StringRef src) { return src.ends_with(".cpp"); }); + return llvm::to_vector( + llvm::concat(libcxx_srcs, libcxxabi_srcs)); } else { static_assert(false, "Invalid runtimes component for an archive runtime builder."); @@ -276,41 +342,57 @@ auto ClangArchiveRuntimesBuilder::CollectSrcFiles() } template - requires(Component == Runtimes::LibUnwind) + requires IsClangArchiveRuntimes auto ClangArchiveRuntimesBuilder::CollectCflags() -> llvm::SmallVector { + llvm::SmallVector cflags; + + // TODO: It would be nice to plumb through an option to enable (some) warnings + // when building runtimes, especially for folks working directly on the Carbon + // toolchain to validate our builds of runtimes. + if constexpr (Component == Runtimes::LibUnwind) { - return { + // TODO: Should libunwind also limit symbol visibility? + cflags = { "-no-canonical-prefixes", + "-D_LIBUNWIND_IS_NATIVE_ONLY", "-O3", "-fPIC", - "-funwind-tables", "-fno-exceptions", "-fno-rtti", + "-funwind-tables", + "-nostdinc++", + "-w", + }; + } else if constexpr (Component == Runtimes::Libcxx) { + cflags = { + "-no-canonical-prefixes", + "-DLIBCXX_BUILDING_LIBCXXABI", + "-D_LIBCPP_BUILDING_LIBRARY", + "-D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES", + "-O3", + "-fPIC", + "-fvisibility-inlines-hidden", + "-fvisibility=hidden", "-nostdinc++", - "-I", - include_path_.native(), - "-D_LIBUNWIND_IS_NATIVE_ONLY", "-w", }; } else { static_assert(false, "Invalid runtimes component for an archive runtime builder."); } + + for (const auto& include_path : include_paths_) { + CARBON_CHECK(include_path.is_absolute(), + "Unexpected relative include path: {0}", include_path); + cflags.append({"-I", include_path.native()}); + } + return cflags; } template - requires(Component == Runtimes::LibUnwind) + requires IsClangArchiveRuntimes auto ClangArchiveRuntimesBuilder::Setup() -> void { - // Symlink the installation's `include` into the runtime. - CARBON_CHECK(include_path_.is_absolute(), - "Unexpected relative include path: {0}", include_path_); - if (auto result = runtimes_builder_->dir().Symlink("include", include_path_); - !result.ok()) { - result_ = std::move(result).error(); - return; - } - // Finish building the runtime once the archive is built. Latch::Handle latch_handle = step_counter_.Init( [this]() mutable { tasks_.async([this]() mutable { Finish(); }); }); @@ -320,7 +402,7 @@ auto ClangArchiveRuntimesBuilder::Setup() -> void { } template - requires(Component == Runtimes::LibUnwind) + requires IsClangArchiveRuntimes auto ClangArchiveRuntimesBuilder::Finish() -> void { CARBON_VLOG("Finished building {0}...\n", archive_path_); if (!archive_->result().ok()) { @@ -332,6 +414,7 @@ auto ClangArchiveRuntimesBuilder::Finish() -> void { } template class ClangArchiveRuntimesBuilder; +template class ClangArchiveRuntimesBuilder; ClangResourceDirBuilder::ClangResourceDirBuilder( ClangRunner* clang, llvm::ThreadPoolInterface* threads, @@ -361,7 +444,7 @@ ClangResourceDirBuilder::ClangResourceDirBuilder( runtimes_builder_ = std::get(std::move(build_dir)); lib_path_ = std::filesystem::path("lib") / target_triple_.str(); archive_.emplace(this, lib_path_ / "libclang_rt.builtins.a", - installation().llvm_runtime_srcs(), + installation().runtimes_root(), CollectBuiltinsSrcFiles(), /*cflags=*/ llvm::SmallVector{ "-no-canonical-prefixes", @@ -498,7 +581,7 @@ auto ClangResourceDirBuilder::BuildCrtFile(llvm::StringRef src_file) (src_file == RuntimeSources::CrtBegin ? "clang_rt.crtbegin.o" : "clang_rt.crtend.o"); std::filesystem::path src_path = - installation().llvm_runtime_srcs() / std::string_view(src_file); + installation().runtimes_root() / std::string_view(src_file); CARBON_VLOG("Building `{0}' from `{1}`...\n", out_path, src_path); bool success = clang_->RunWithNoRuntimes({ diff --git a/toolchain/driver/clang_runtimes.h b/toolchain/driver/clang_runtimes.h index 2aa6f2770373a..0ff5eea894b45 100644 --- a/toolchain/driver/clang_runtimes.h +++ b/toolchain/driver/clang_runtimes.h @@ -107,24 +107,21 @@ class ClangRuntimesBuilderBase::ArchiveBuilder { // - `archive_path` is the _relative_ path of the archive file within the // built // runtimes directory. - // - `srcs_path` is the _absolute_ root of source files used to build the - // archive. - // - `src_files` is a list of the file paths to build into the archive, - // relative to the `srcs_path`. These are `StringRef`s so they can reference - // constant `StringLiteral`s in common cases. + // - `src_files` is a list of the _absolute_ file paths to build into the + // archive. // - `cflags` are the compile flags that should be used for all the compiles // in this archive. ArchiveBuilder(ClangRuntimesBuilderBase* builder, std::filesystem::path archive_path, - std::filesystem::path srcs_path, - llvm::ArrayRef src_files, - llvm::ArrayRef cflags) + std::filesystem::path srcs_root, + llvm::SmallVector src_files, + llvm::SmallVector cflags) : builder_(builder), vlog_stream_(builder_->vlog_stream_), archive_path_(std::move(archive_path)), - srcs_path_(std::move(srcs_path)), - src_files_(src_files), - cflags_(cflags) {} + srcs_root_(std::move(srcs_root)), + src_files_(std::move(src_files)), + cflags_(std::move(cflags)) {} // Start building the archive, with a latch handle to signal its completion. // @@ -182,7 +179,7 @@ class ClangRuntimesBuilderBase::ArchiveBuilder { std::filesystem::path archive_path_; - std::filesystem::path srcs_path_; + std::filesystem::path srcs_root_; llvm::SmallVector src_files_; llvm::SmallVector cflags_; @@ -203,6 +200,11 @@ class ClangRuntimesBuilderBase::ArchiveBuilder { ErrorOr result_ = Error("No archive built!"); }; +template +concept IsClangArchiveRuntimes = requires { + requires(Component == Runtimes::LibUnwind || Component == Runtimes::Libcxx); +}; + // A class template to build runtimes consisting of a single archive. // // The template argument comes from the `Runtimes::Component` enum, but is only @@ -210,7 +212,7 @@ class ClangRuntimesBuilderBase::ArchiveBuilder { // requires to enforce that the components used are exactly one of those // supported so we can also move instantiation into the `.cpp` file. template - requires(Component == Runtimes::LibUnwind) + requires IsClangArchiveRuntimes class ClangArchiveRuntimesBuilder : public ClangRuntimesBuilderBase { public: // Constructing this class will attempt to build the `Component` archive into @@ -241,23 +243,22 @@ class ClangArchiveRuntimesBuilder : public ClangRuntimesBuilderBase { // directory. auto Finish() -> void; - // The root path used for any of the source files. - std::filesystem::path srcs_path_; - - // The (absolute) include path used during the compilation of the source - // files. - std::filesystem::path include_path_; - // The relative archive path within the runtimes' build directory. std::filesystem::path archive_path_; + // The (absolute) include paths used during the compilation of the source + // files. + llvm::SmallVector include_paths_; + // The archive builder if it is necessary to build the archive. std::optional archive_; }; extern template class ClangArchiveRuntimesBuilder; +extern template class ClangArchiveRuntimesBuilder; using LibunwindBuilder = ClangArchiveRuntimesBuilder; +using LibcxxBuilder = ClangArchiveRuntimesBuilder; // Builds the target-specific resource directory for Clang. // diff --git a/toolchain/driver/clang_runtimes_test.cpp b/toolchain/driver/clang_runtimes_test.cpp index 551af9884c207..588f9fc2f59dc 100644 --- a/toolchain/driver/clang_runtimes_test.cpp +++ b/toolchain/driver/clang_runtimes_test.cpp @@ -198,5 +198,41 @@ TEST_F(ClangRuntimesTest, Libunwind) { ExpectSymbol(libunwind_symbols, "__unw_get_proc_info"); } +TEST_F(ClangRuntimesTest, Libcxx) { +#if __has_feature(address_sanitizer) + // ASan causes Clang and LLVM to be _egregiously_ inefficient at compiling + // libc++, taking 5x - 10x longer than without ASan. Rough estimate is that it + // would take 5-10 minutes on GitHub's Linux runner. Given the limited utility + // of this test coverage, skip it in that configuration. This also misses + // assert-coverage for building libc++, but we don't really expect issues + // there. Misconfiguration and other common issues should still be covered in + // fully optimized builds at much lower cost. + GTEST_SKIP() << "Skipping build of libc++ with an ASan-itized Clang"; +#endif + + LibcxxBuilder libcxx_builder(&runner_, &threads_, target_triple_, &runtimes_); + auto build_result = std::move(libcxx_builder).Wait(); + ASSERT_TRUE(build_result.ok()) << build_result.error(); + std::filesystem::path runtimes_path = std::move(*build_result); + + std::filesystem::path libcxx_path = runtimes_path / "lib/libc++.a"; + std::string libcxx_symbols = NmListDefinedSymbols(libcxx_path); + + // First check a few fundamental symbols from libc++.a, including symbols both + // within the ABI namespace and outside of it. + ExpectSymbol(libcxx_symbols, "_ZNKSt12bad_any_cast4whatEv"); + ExpectSymbol(libcxx_symbols, "_ZNSt2_C8to_charsEPcS0_d"); + ExpectSymbol(libcxx_symbols, "_ZSt17current_exceptionv"); + ExpectSymbol(libcxx_symbols, "_ZNKSt2_C10filesystem4path10__filenameEv"); + + // Check that several of the libc++abi object files are also included in the + // archive. + ExpectSymbol(libcxx_symbols, "__cxa_bad_cast"); + ExpectSymbol(libcxx_symbols, "__cxa_new_handler"); + ExpectSymbol(libcxx_symbols, "__cxa_demangle"); + ExpectSymbol(libcxx_symbols, "__cxa_get_globals"); + ExpectSymbol(libcxx_symbols, "_ZSt9terminatev"); +} + } // namespace } // namespace Carbon diff --git a/toolchain/driver/runtimes_cache.h b/toolchain/driver/runtimes_cache.h index e499af1eaf85f..7bbd20f806943 100644 --- a/toolchain/driver/runtimes_cache.h +++ b/toolchain/driver/runtimes_cache.h @@ -47,6 +47,7 @@ class Runtimes { enum Component { ClangResourceDir, LibUnwind, + Libcxx, NumComponents, }; @@ -139,6 +140,8 @@ class Runtimes { return "clang_resource_dir"; case LibUnwind: return "libunwind"; + case Libcxx: + return "libcxx"; case NumComponents: CARBON_FATAL("Invalid component"); } diff --git a/toolchain/install/BUILD b/toolchain/install/BUILD index acb92291faf07..5a0b343bae382 100644 --- a/toolchain/install/BUILD +++ b/toolchain/install/BUILD @@ -6,11 +6,12 @@ load( "@llvm-project//:vars.bzl", "LLVM_VERSION_MAJOR", ) -load("@rules_python//python:defs.bzl", "py_test") +load("@rules_python//python:defs.bzl", "py_binary", "py_test") load("//bazel/cc_rules:defs.bzl", "cc_binary", "cc_library", "cc_test") load("//bazel/manifest:defs.bzl", "manifest") load("//toolchain/base:llvm_tools.bzl", "LLVM_MAIN_TOOLS", "LLVM_TOOL_ALIASES") load("//toolchain/base:runtime_sources.bzl", "BUILTINS_FILEGROUPS", "CRT_FILES") +load("configure_cmake_file.bzl", "configure_cmake_file") load("install_filegroups.bzl", "install_filegroup", "install_symlink", "install_target", "make_install_filegroups") load("pkg_helpers.bzl", "pkg_naming_variables", "pkg_tar_and_test") @@ -162,6 +163,175 @@ filegroup( srcs = CRT_FILES.values() + BUILTINS_FILEGROUPS.values(), ) +py_binary( + name = "configure_cmake_file_impl", + srcs = ["configure_cmake_file_impl.py"], +) + +configure_cmake_file( + name = "libcxx_site_config_gen", + src = "@llvm-project//libcxx:include/__config_site.in", + out = "staging_libcxx/include/__config_site", + defines = { + # We can inject custom logic at the end of the site configuration with + # the ABI defines. This can custom set (or re-set) any of the relevant + # configuration defines. Note that while this is sorted here in the + # BUILD file, it is expanded at the _end_ of the configuration header + # and so overrides the other configuration settings. + # + # TODO: This is a lot of C++ code to embed into a BUILD file. Even + # though it moves it farther from the interacting CMake defines, we + # should look at factoring this into a header that is included. + "_LIBCPP_ABI_DEFINES": "\n".join([ + # We want to install a single header that works in all build modes, + # so we define the ABI namespace based on how the header is used + # rather than a fixed one. However, we only support use with Clang + # and so we assume `__has_feature` is available and works. + # + # Note that generally, we don't rely on different ABI namespaces for + # functionality -- the distinction is more to make errors when + # linking with the wrong build of the standard library obvious and + # immediate. We only can achieve this for sanitizers that have a + # preprocessor detectable model. + "#if __has_feature(address_sanitizer)", + "# undef _LIBCPP_ABI_NAMESPACE", + "# define _LIBCPP_ABI_NAMESPACE __asan", + # Also mark that libc++ will be instrumented. + "# undef _LIBCPP_INSTRUMENTED_WITH_ASAN", + "# define _LIBCPP_INSTRUMENTED_WITH_ASAN 1", + "#elif __has_feature(memory_sanitizer)", + # TODO: If a track-origins macro becomes available, we should + # distinguish that case, too. + "# undef _LIBCPP_ABI_NAMESPACE", + "# define _LIBCPP_ABI_NAMESPACE __msan", + "#elif __has_feature(thread_sanitizer)", + "# undef _LIBCPP_ABI_NAMESPACE", + "# define _LIBCPP_ABI_NAMESPACE __tsan", + "#elif __has_feature(cfi_sanitizer)", + "# undef _LIBCPP_ABI_NAMESPACE", + "# define _LIBCPP_ABI_NAMESPACE __cfi", + "#endif", + "", + + # Establish a default hardening mode where possible. + "#ifndef _LIBCPP_HARDENING_MODE", + "# ifndef NDEBUG", + # !NDEBUG has significant overhead anyway and is explicitly a + # debugging build rather than a production build. + "# define _LIBCPP_HARDENING_MODE _LIBCPP_HARDENING_MODE_DEBUG", + "# else", + # Default to the fast hardening checks. + "# define _LIBCPP_HARDENING_MODE _LIBCPP_HARDENING_MODE_FAST", + "# endif", + "#endif", + "", + + # CUDA can't call any existing abort implementations, so disable + # hardening there. + "#ifdef __CUDA__", + "# undef _LIBCPP_HARDENING_MODE", + "# define _LIBCPP_HARDENING_MODE _LIBCPP_HARDENING_MODE_NONE", + "#endif", + "", + + # Setup platform-dependent features using preprocessor logic. + "#ifdef __linux__", + "# undef _LIBCPP_HAS_TIME_ZONE_DATABASE", + "# define _LIBCPP_HAS_TIME_ZONE_DATABASE 1", + "#endif", + "", + + # Mixing translation units compiled with different versions of + # libc++ is unsupported. Disable ABI tags to decrease symbol + # lengths. + "#define _LIBCPP_NO_ABI_TAG", + ]), + + # No forced ABI. + "_LIBCPP_ABI_FORCE_ITANIUM": "OFF", + "_LIBCPP_ABI_FORCE_MICROSOFT": "OFF", + + # We use the unstable ABI and define a custom, Carbon-specific ABI + # namespace. This also matches the mangling prefix used for Carbon + # symbols. + "_LIBCPP_ABI_NAMESPACE": "_C", + # TODO: Fix the need to define _LIBCPP_ABI_VERSION when the unstable + # ABI is selected. + "_LIBCPP_ABI_VERSION": "999", + + # Follow hardening mode for the assertion semantics. + "_LIBCPP_ASSERTION_SEMANTIC_DEFAULT": "_LIBCPP_ASSERTION_SEMANTIC_HARDENING_DEPENDENT", + + # Enable various features in libc++ available across platforms. We + # describe these in a block to allow the BUILD file to sort them. + # + # - We enable threads, and use auto-detection rather than forcing an + # API. + # - Availability annotations do not apply to Carbon's libc++, so those + # are disabled. + # + # Where there are platform differences in the features, we disable them + # here and re-enable them in the `_LIBCPP_ABI_DEFINES` section using + # custom logic to detect the relevant platform. + "_LIBCPP_HAS_FILESYSTEM": "ON", + "_LIBCPP_HAS_LOCALIZATION": "ON", + "_LIBCPP_HAS_MONOTONIC_CLOCK": "ON", + "_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS": "ON", + "_LIBCPP_HAS_RANDOM_DEVICE": "ON", + "_LIBCPP_HAS_TERMINAL": "ON", + "_LIBCPP_HAS_THREADS": "ON", + "_LIBCPP_HAS_THREAD_API_EXTERNAL": "OFF", + "_LIBCPP_HAS_THREAD_API_PTHREAD": "OFF", + "_LIBCPP_HAS_THREAD_API_WIN32": "OFF", + "_LIBCPP_HAS_TIME_ZONE_DATABASE": "OFF", + "_LIBCPP_HAS_UNICODE": "ON", + "_LIBCPP_HAS_VENDOR_AVAILABILITY_ANNOTATIONS": "OFF", + "_LIBCPP_HAS_WIDE_CHARACTERS": "ON", + + # When ASan is enabled, we ensure that libc++ is built with it as well. + # However, we can't set this more carefully here so we set it to off and + # override it below when using ASan. + "_LIBCPP_INSTRUMENTED_WITH_ASAN": "OFF", + + # Set the parallel backend to serial. + # TODO: We should revisit this. + "_LIBCPP_PSTL_BACKEND_SERIAL": "1", + }, +) + +configure_cmake_file( + name = "libcxx_assertion_handler_gen", + src = "@llvm-project//libcxx:vendor/llvm/default_assertion_handler.in", + out = "staging_libcxx/include/__assertion_handler", + defines = { + # Currently the default handler needs no substitutions. + }, +) + +filegroup( + name = "libcxx", + srcs = [ + "@llvm-project//libcxx:libcxx_hdrs", + "@llvm-project//libcxx:libcxx_srcs", + ], +) + +filegroup( + name = "libcxx_gen_files", + srcs = [ + "staging_libcxx/include/__assertion_handler", + "staging_libcxx/include/__config_site", + ], +) + +filegroup( + name = "libcxxabi", + srcs = [ + "@llvm-project//libcxxabi:libcxxabi_hdrs", + "@llvm-project//libcxxabi:libcxxabi_srcs", + ], +) + filegroup( name = "libunwind", srcs = [ @@ -170,6 +340,20 @@ filegroup( ], ) +# Currently, we're only installing the subset of LLVM's libc internals needed to +# build libc++. At some point, we should ship LLVM's libc itself, and that will +# likely expand this to cover more of the source. However, we'll still want to +# distinguish between the _internal_ installation and the generated set of +# headers that we inject into the include search for user compiles. The +# `include` subdirectory in this file group is _not_ intended to be exposed to +# user compiles, only to compilation of runtimes. +filegroup( + name = "libc_internal", + srcs = [ + "@llvm-project//libc:libcxx_shared_headers_hdrs", + ], +) + # Given a root `prefix_root`, the hierarchy looks like: # # - prefix_root/bin: Binaries intended for direct use. @@ -202,14 +386,35 @@ install_dirs = { executable = True, is_driver = True, ), + # TODO: Consider if we want to keep `core` here or group it with + # runtimes. It is a bit of both -- standard library, and runtimes. install_filegroup("core", "//core:prelude"), - install_filegroup("libunwind", ":libunwind"), ], "lib/carbon/llvm/bin": [install_symlink( name, "../../carbon-busybox", is_driver = True, ) for name in llvm_binaries], + "lib/carbon/runtimes": [ + install_filegroup( + "builtins", + ":clang_builtins_runtimes", + remove_prefix = "lib/builtins/", + ), + install_filegroup("libcxx", ":libcxx"), + install_filegroup("libcxxabi", ":libcxxabi"), + install_filegroup("libunwind", ":libunwind"), + ], + "lib/carbon/runtimes/libc": [ + install_filegroup("internal", ":libc_internal"), + ], + "lib/carbon/runtimes/libcxx": [ + install_filegroup( + "include", + ":libcxx_gen_files", + remove_prefix = "staging_libcxx/include/", + ), + ], "lib/carbon/llvm/lib/clang/" + LLVM_VERSION_MAJOR: [ install_filegroup( "include", @@ -218,9 +423,6 @@ install_dirs = { remove_prefix = "staging/include/", ), ], - "lib/carbon/llvm/lib/clang/" + LLVM_VERSION_MAJOR + "/src": [ - install_filegroup("builtins", ":clang_builtins_runtimes", "lib/builtins/"), - ], } make_install_filegroups( diff --git a/toolchain/install/configure_cmake_file.bzl b/toolchain/install/configure_cmake_file.bzl new file mode 100644 index 0000000000000..196f6d99d7490 --- /dev/null +++ b/toolchain/install/configure_cmake_file.bzl @@ -0,0 +1,72 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +"""A Starlark implementation of a CMake-like configure_file rule.""" + +def _configure_cmake_file_impl(ctx): + """Implementation for the configure_cmake_file rule.""" + + # Flatten the defines dictionary into a list of command-line arguments + # for the implementation script: + # + # ["--defines", KEY1, VAL1, "--defines", KEY2, VAL2] + define_args = [] + for key, value in ctx.attr.defines.items(): + define_args.append("--defines") + define_args.append(key) + define_args.append(value) + + ctx.actions.run( + executable = ctx.executable._impl_script, + arguments = [ + "--src", + ctx.file.src.path, + "--out", + ctx.outputs.out.path, + ] + define_args, + inputs = depset([ctx.file.src, ctx.executable._impl_script]), + outputs = [ctx.outputs.out], + mnemonic = "ConfigureCmakeFile", + progress_message = "Configuring file: %{label}", + ) + + return [DefaultInfo(files = depset([ctx.outputs.out]))] + +configure_cmake_file = rule( + implementation = _configure_cmake_file_impl, + attrs = { + "defines": attr.string_dict( + mandatory = True, + doc = "A dictionary of key-value definitions to substitute.", + ), + "out": attr.output( + mandatory = True, + doc = "The generated output file.", + ), + "src": attr.label( + allow_single_file = True, + mandatory = True, + doc = "The input '.in' template file.", + ), + "_impl_script": attr.label( + default = Label("//toolchain/install:configure_cmake_file_impl"), + allow_files = True, + executable = True, + cfg = "exec", + ), + }, + doc = """ +A rule that performs CMake-style configuration of an input file. + +This rule processes an input file (`.in`) and generates an output file +based on a dictionary of definitions. It provides emulation +of the most commonly used aspects of CMake's `configure_file` command: +https://cmake.org/cmake/help/latest/command/configure_file.html + +Notable aspects not implemented are the following: + +* Substitution of cache values using `$CACHE{VAR}` syntax. +* Substitution of environment variables using `$ENV{VAR}` syntax. +""", +) diff --git a/toolchain/install/configure_cmake_file_impl.py b/toolchain/install/configure_cmake_file_impl.py new file mode 100644 index 0000000000000..01a69077f379c --- /dev/null +++ b/toolchain/install/configure_cmake_file_impl.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +__copyright__ = """ +Part of the Carbon Language project, under the Apache License v2.0 with LLVM +Exceptions. See /LICENSE for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +"""Script to apply a set of defines to a CMake-style configure file. + +This serves as the action implementation for `configure_cmake_file.bzl`. See the +documentation in the rule of that file for more details about how to use this, +or `--help` on the script. +""" + +import argparse +import re +from typing import Dict + +# A set of CMake values that are considered "false". +# Based on https://cmake.org/cmake/help/latest/command/if.html +_CMAKE_FALSE_VALUES = { + "", + "0", + "OFF", + "NO", + "N", + "FALSE", + "IGNORE", + "NOTFOUND", +} + +_VAR_AT_PATTERN = re.compile(r"@([^@]*)@") +_VAR_DOLLAR_PATTERN = re.compile(r"${([^}]*)}") + +_DIRECTIVE_PATTERN = re.compile( + r"^#(?P[ \t]*)cmakedefine\s+(?P\w+)(?P.*)?$" +) +_DIRECTIVE_01_PATTERN = re.compile( + r"^#(?P[ \t]*)cmakedefine01\s+(?P\w+)$" +) + + +def _is_cmake_true(value: str) -> bool: + """Returns true if the value is not a CMake false value. + + This is how CMake defines values as 'true' vs. 'false': + https://cmake.org/cmake/help/latest/command/if.html + """ + return ( + value.upper() not in _CMAKE_FALSE_VALUES + and not value.upper().endswith("-NOTFOUND") + ) + + +def _substitute_variables(text: str, defines: Dict[str, str]) -> str: + """Substitutes @VAR@ and ${VAR} style variables in a string.""" + + def repl(m: re.Match) -> str: + return defines.get(str(m.group(1)), "") + + return re.sub( + _VAR_AT_PATTERN, repl, re.sub(_VAR_DOLLAR_PATTERN, repl, text) + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--src", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--defines", nargs=2, action="append", default=[]) + args = parser.parse_args() + + defines = dict(args.defines) + + with open(args.src, "r") as f: + content = f.read() + + output_lines = [] + for line in content.splitlines(): + if m := re.match(_DIRECTIVE_PATTERN, line): + var = m.group("var") + if var in defines and _is_cmake_true(defines[var]): + rest = _substitute_variables(m.group("rest"), defines) + output_lines.append( + "#%sdefine %s %s" % (m.group("indent"), var, rest) + ) + else: + # The variable is false, so leave it undefined. + output_lines.append("/* #undef %s */" % var) + elif m := re.match(_DIRECTIVE_01_PATTERN, line): + var = m.group("var") + indent = m.group("indent") + if var in defines and _is_cmake_true(defines[var]): + output_lines.append("#%sdefine %s 1" % (indent, var)) + else: + output_lines.append("#%sdefine %s 0" % (indent, var)) + else: + output_lines.append(_substitute_variables(line, defines)) + + with open(args.out, "w") as f: + f.write("\n".join(output_lines) + "\n") + + +if __name__ == "__main__": + main() diff --git a/toolchain/install/install_paths.cpp b/toolchain/install/install_paths.cpp index fab436a313010..ead703431cd7c 100644 --- a/toolchain/install/install_paths.cpp +++ b/toolchain/install/install_paths.cpp @@ -219,15 +219,29 @@ auto InstallPaths::clang_resource_path() const -> std::filesystem::path { return prefix_ / "lib/carbon/llvm/lib/clang/" CLANG_VERSION_MAJOR_STRING; } -auto InstallPaths::llvm_runtime_srcs() const -> std::filesystem::path { +auto InstallPaths::runtimes_root() const -> std::filesystem::path { // TODO: Adjust this to work equally well on Windows. - return prefix_ / "lib/carbon/llvm/lib/clang/" CLANG_VERSION_MAJOR_STRING - "/src"; + return prefix_ / "lib/carbon/runtimes"; } auto InstallPaths::libunwind_path() const -> std::filesystem::path { // TODO: Adjust this to work equally well on Windows. - return prefix_ / "lib/carbon/libunwind"; + return prefix_ / "lib/carbon/runtimes/libunwind"; +} + +auto InstallPaths::libcxx_path() const -> std::filesystem::path { + // TODO: Adjust this to work equally well on Windows. + return prefix_ / "lib/carbon/runtimes/libcxx"; +} + +auto InstallPaths::libcxxabi_path() const -> std::filesystem::path { + // TODO: Adjust this to work equally well on Windows. + return prefix_ / "lib/carbon/runtimes/libcxxabi"; +} + +auto InstallPaths::libc_path() const -> std::filesystem::path { + // TODO: Adjust this to work equally well on Windows. + return prefix_ / "lib/carbon/runtimes/libc"; } auto InstallPaths::digest_path() const -> std::filesystem::path { diff --git a/toolchain/install/install_paths.h b/toolchain/install/install_paths.h index 9233db9a05c66..1ba1c1e49ca36 100644 --- a/toolchain/install/install_paths.h +++ b/toolchain/install/install_paths.h @@ -108,12 +108,21 @@ class InstallPaths { // The path to the Clang resources. auto clang_resource_path() const -> std::filesystem::path; - // The path to the root of LLVM runtime sources. - auto llvm_runtime_srcs() const -> std::filesystem::path; + // The path to the root of the runtimes. + auto runtimes_root() const -> std::filesystem::path; // The path to `libunwind` runtime. auto libunwind_path() const -> std::filesystem::path; + // The path to `libunwind` runtime. + auto libcxx_path() const -> std::filesystem::path; + + // The path to `libunwind` runtime. + auto libcxxabi_path() const -> std::filesystem::path; + + // The path to the LLVM `libc` runtime. + auto libc_path() const -> std::filesystem::path; + // The installation digest path. // // This file contains a digest of the installation.