Introduce a runtimes caching and management layer (#6002)

This layer allows runtimes to be built on-demand but cached in a
consistent and re-usable location on the system. It handles careful
filesystem operations to ensure consistency even in the face of multiple
versions and build configurations.

This addresses a number of TODOs from the initial runtimes building
on-demand, and sets the stage to scale up to more runtimes.

This doesn't switch on-demand runtimes to be on by default, I wanted to
wait and make that change as a separate step.

---------

Co-authored-by: Geoff Romer <[email protected]>

Author: chandlerc

common/filesystem.cpp

@@ -62,7 +62,10 @@ auto PathError::Print(llvm::raw_ostream& out) const -> void {
   // The `format_` member is a `StringLiteral` that is null terminated, so
   // `.data()` is safe here.
   // NOLINTNEXTLINE(bugprone-suspicious-stringview-data-usage)
-  out << llvm::formatv(format_.data(), path_, dir_fd_) << " failed: ";
+  out << llvm::formatv(format_.data(), path_,
+                       dir_fd_ == AT_FDCWD ? std::string("AT_FDCWD")
+                                           : std::to_string(dir_fd_))
+      << " failed: ";
   PrintErrorNumber(out, unix_errnum());
 }
 

toolchain/diagnostics/diagnostic_kind.def

@@ -22,6 +22,8 @@
 // ============================================================================
 
 CARBON_DIAGNOSTIC_KIND(DriverInstallInvalid)
+CARBON_DIAGNOSTIC_KIND(DriverRuntimesCacheInvalid)
+CARBON_DIAGNOSTIC_KIND(DriverPrebuiltRuntimesInvalid)
 CARBON_DIAGNOSTIC_KIND(DriverCommandLineParseFailed)
 CARBON_DIAGNOSTIC_KIND(CompilePhaseFlagConflict)
 CARBON_DIAGNOSTIC_KIND(CompilePreludeManifestError)

toolchain/driver/BUILD

@@ -23,6 +23,7 @@ cc_library(
     hdrs = ["clang_runner.h"],
     deps = [
         ":llvm_runner",
+        ":runtimes_cache",
         ":tool_runner_base",
         "//common:error",
         "//common:filesystem",
@@ -130,6 +131,7 @@ cc_library(
         ":clang_runner",
         ":lld_runner",
         ":llvm_runner",
+        ":runtimes_cache",
         "//common:command_line",
         "//common:error",
         "//common:ostream",
@@ -270,6 +272,44 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "runtimes_cache",
+    srcs = ["runtimes_cache.cpp"],
+    hdrs = ["runtimes_cache.h"],
+    deps = [
+        "//common:check",
+        "//common:error",
+        "//common:filesystem",
+        "//common:ostream",
+        "//common:version",
+        "//common:vlog",
+        "//toolchain/install:install_paths",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
+cc_test(
+    name = "runtimes_cache_test",
+    size = "small",
+    srcs = ["runtimes_cache_test.cpp"],
+    data = ["//toolchain/install:install_data"],
+    deps = [
+        ":runtimes_cache",
+        "//common:check",
+        "//common:error_test_helpers",
+        "//common:filesystem",
+        "//common:ostream",
+        "//common:raw_string_ostream",
+        "//common:version",
+        "//testing/base:capture_std_streams",
+        "//testing/base:file_helpers",
+        "//testing/base:global_exe_path",
+        "//testing/base:gtest_main",
+        "@googletest//:gtest",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
 cc_library(
     name = "tool_runner_base",
     srcs = ["tool_runner_base.cpp"],

toolchain/driver/build_runtimes_subcommand.cpp

@@ -4,6 +4,8 @@
 
 #include "toolchain/driver/build_runtimes_subcommand.h"
 
+#include <variant>
+
 #include "llvm/TargetParser/Triple.h"
 #include "toolchain/driver/clang_runner.h"
 
@@ -43,9 +45,6 @@ BuildRuntimesSubcommand::BuildRuntimesSubcommand()
     : DriverSubcommand(SubcommandInfo) {}
 
 auto BuildRuntimesSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
-  ClangRunner runner(driver_env.installation, driver_env.fs,
-                     driver_env.vlog_stream);
-
   // Don't run Clang when fuzzing, it is known to not be reliable under fuzzing
   // due to many unfixed issues.
   if (TestAndDiagnoseIfFuzzingExternalLibraries(driver_env, "clang")) {
@@ -56,39 +55,44 @@ auto BuildRuntimesSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
   CARBON_DIAGNOSTIC(FailureBuildingRuntimes, Error,
                     "failure building runtimes: {0}", std::string);
 
-  auto tmp_result = Filesystem::MakeTmpDir();
-  if (!tmp_result.ok()) {
+  auto run_result = RunInternal(driver_env);
+  if (!run_result.ok()) {
     driver_env.emitter.Emit(FailureBuildingRuntimes,
-                            tmp_result.error().message());
+                            run_result.error().message());
     return {.success = false};
   }
-  Filesystem::RemovingDir tmp_dir = *std::move(tmp_result);
-
-  // TODO: Currently, the default location is just a subdirectory of the
-  // temporary directory used for the build. This allows the subcommand to be
-  // used to test and debug runtime building, but not for the results to be
-  // reused. Eventually, this should be connected to the same runtimes cache
-  // used by link commands.
-  std::filesystem::path output_path =
-      options_.directory.empty()
-          ? tmp_dir.abs_path() / "runtimes"
-          : std::filesystem::path(options_.directory.str());
-
-  // Hard code a subdirectory of the runtimes output for the Clang resource
-  // directory runtimes.
-  //
-  // TODO: This should be replaced with an abstraction that manages the layout
-  // of the generated runtimes rather than hardcoding it.
-  std::filesystem::path resource_dir_path = output_path / "clang_resource_dir";
-
-  auto build_result = runner.BuildTargetResourceDir(
-      options_.codegen_options.target, resource_dir_path, tmp_dir.abs_path());
-  if (!build_result.ok()) {
-    driver_env.emitter.Emit(FailureBuildingRuntimes,
-                            build_result.error().message());
+
+  llvm::outs() << "Built runtimes: " << *run_result << "\n";
+  return {.success = true};
+}
+
+auto BuildRuntimesSubcommand::RunInternal(DriverEnv& driver_env)
+    -> ErrorOr<std::filesystem::path> {
+  ClangRunner runner(driver_env.installation, &driver_env.runtimes_cache,
+                     driver_env.fs, driver_env.vlog_stream);
+
+  Runtimes::Cache::Features features = {
+      .target = options_.codegen_options.target.str()};
+
+  bool is_cache = options_.directory.empty();
+  std::filesystem::path explicit_output_path = options_.directory.str();
+  if (!is_cache) {
+    auto access_result = Filesystem::Cwd().Access(explicit_output_path);
+    if (access_result.ok()) {
+      return Error("output directory already exists");
+    }
+    if (!access_result.error().no_entity()) {
+      return std::move(access_result).error();
+    }
   }
 
-  return {.success = build_result.ok()};
+  CARBON_ASSIGN_OR_RETURN(
+      auto runtimes,
+      is_cache ? driver_env.runtimes_cache.Lookup(features)
+               : Runtimes::Make(explicit_output_path, driver_env.vlog_stream));
+  CARBON_ASSIGN_OR_RETURN(auto tmp_dir, Filesystem::MakeTmpDir());
+
+  return runner.BuildTargetResourceDir(features, runtimes, tmp_dir.abs_path());
 }
 
 }  // namespace Carbon

toolchain/driver/build_runtimes_subcommand.h

@@ -36,6 +36,8 @@ class BuildRuntimesSubcommand : public DriverSubcommand {
   auto Run(DriverEnv& driver_env) -> DriverResult override;
 
  private:
+  auto RunInternal(DriverEnv& driver_env) -> ErrorOr<std::filesystem::path>;
+
   BuildRuntimesOptions options_;
 };
 

toolchain/driver/clang_runner.cpp

@@ -14,6 +14,7 @@
 #include <string>
 #include <system_error>
 #include <utility>
+#include <variant>
 
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticOptions.h"
@@ -51,10 +52,12 @@ auto clang_main(int Argc, char** Argv, const llvm::ToolContext& ToolContext)
 namespace Carbon {
 
 ClangRunner::ClangRunner(const InstallPaths* install_paths,
+                         Runtimes::Cache* runtimes_cache,
                          llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,
                          llvm::raw_ostream* vlog_stream,
                          bool build_runtimes_on_demand)
     : ToolRunnerBase(install_paths, vlog_stream),
+      runtimes_cache_(runtimes_cache),
       fs_(std::move(fs)),
       diagnostic_ids_(new clang::DiagnosticIDs()),
       build_runtimes_on_demand_(build_runtimes_on_demand) {}
@@ -130,47 +133,44 @@ static auto IsNonLinkCommand(llvm::ArrayRef<llvm::StringRef> args) -> bool {
   });
 }
 
-auto ClangRunner::Run(
-    llvm::ArrayRef<llvm::StringRef> args,
-    std::optional<std::filesystem::path> prebuilt_resource_dir_path)
-    -> ErrorOr<bool> {
+auto ClangRunner::Run(llvm::ArrayRef<llvm::StringRef> args,
+                      Runtimes* prebuilt_runtimes) -> ErrorOr<bool> {
   // Check the args to see if we have a known target-independent command. If so,
   // directly dispatch it to avoid the cost of building the target resource
   // directory.
   // TODO: Maybe handle response file expansion similar to the Clang CLI?
   if (args.empty() || args[0].starts_with("-cc1") || IsNonLinkCommand(args) ||
-      (!build_runtimes_on_demand_ && !prebuilt_resource_dir_path)) {
+      (!build_runtimes_on_demand_ && !prebuilt_runtimes)) {
     return RunTargetIndependentCommand(args);
   }
 
   std::string target = ComputeClangTarget(args);
 
   // If we have pre-built runtimes, use them rather than building on demand.
-  if (prebuilt_resource_dir_path) {
-    return RunInternal(args, target, prebuilt_resource_dir_path->native());
+  if (prebuilt_runtimes) {
+    CARBON_ASSIGN_OR_RETURN(std::filesystem::path prebuilt_resource_dir_path,
+                            prebuilt_runtimes->Get(Runtimes::ClangResourceDir));
+    return RunInternal(args, target, prebuilt_resource_dir_path.native());
   }
   CARBON_CHECK(build_runtimes_on_demand_);
 
   // Otherwise, we need to build a target resource directory.
-  //
-  // TODO: Currently, this builds the runtimes in a temporary directory that is
-  // removed after the Clang invocation. That requires building them on each
-  // execution which is expensive and slow. Eventually, we want to replace this
-  // with using an on-disk cache so that only the first execution has to build
-  // the runtimes and subsequently the cached build can be used.
   CARBON_VLOG("Building target resource dir...\n");
-  CARBON_ASSIGN_OR_RETURN(Filesystem::RemovingDir tmp_dir,
-                          Filesystem::MakeTmpDir());
-
-  // Hard code the subdirectory for the resource-dir runtimes.
-  //
-  // TODO: This should be replaced with an abstraction that manages the layout
-  // of a built runtimes tree.
-  std::filesystem::path resource_dir_path =
-      tmp_dir.abs_path() / "clang_resource_dir";
-
-  CARBON_RETURN_IF_ERROR(
-      BuildTargetResourceDir(target, resource_dir_path, tmp_dir.abs_path()));
+  Runtimes::Cache::Features features = {.target = target};
+  CARBON_ASSIGN_OR_RETURN(Runtimes runtimes, runtimes_cache_->Lookup(features));
+
+  // We need to build the Clang resource directory for these runtimes. This
+  // requires a temporary directory as well as the destination directory for
+  // the build. The temporary directory should only be used during the build,
+  // not once we are running Clang with the built runtime.
+  std::filesystem::path resource_dir_path;
+  {
+    CARBON_ASSIGN_OR_RETURN(Filesystem::RemovingDir tmp_dir,
+                            Filesystem::MakeTmpDir());
+    CARBON_ASSIGN_OR_RETURN(
+        resource_dir_path,
+        BuildTargetResourceDir(features, runtimes, tmp_dir.abs_path()));
+  }
 
   // Note that this function always successfully runs `clang` and returns a bool
   // to indicate whether `clang` itself succeeded, not whether the runner was
@@ -186,32 +186,37 @@ auto ClangRunner::RunTargetIndependentCommand(
 }
 
 auto ClangRunner::BuildTargetResourceDir(
-    llvm::StringRef target, const std::filesystem::path& resource_dir_path,
-    const std::filesystem::path& tmp_path) -> ErrorOr<Success> {
+    const Runtimes::Cache::Features& features, Runtimes& runtimes,
+    const std::filesystem::path& tmp_path) -> ErrorOr<std::filesystem::path> {
   // Disable any leaking of memory while building the target resource dir, and
   // restore the previous setting at the end.
   auto restore_leak_flag = llvm::make_scope_exit(
       [&, orig_flag = enable_leaking_] { enable_leaking_ = orig_flag; });
   enable_leaking_ = false;
 
-  // Create the destination directory if needed.
-  CARBON_ASSIGN_OR_RETURN(
-      Filesystem::Dir resource_dir,
-      Filesystem::Cwd().CreateDirectories(resource_dir_path));
+  CARBON_ASSIGN_OR_RETURN(auto build_dir,
+                          runtimes.Build(Runtimes::ClangResourceDir));
+  if (std::holds_alternative<std::filesystem::path>(build_dir)) {
+    // Found cached build.
+    return std::get<std::filesystem::path>(std::move(build_dir));
+  }
+
+  auto builder = std::get<Runtimes::Builder>(std::move(build_dir));
+  std::string target = features.target;
 
   // Symlink the installation's `include` and `share` directories.
   std::filesystem::path install_resource_path =
       installation_->clang_resource_path();
   CARBON_RETURN_IF_ERROR(
-      resource_dir.Symlink("include", install_resource_path / "include"));
+      builder.dir().Symlink("include", install_resource_path / "include"));
   CARBON_RETURN_IF_ERROR(
-      resource_dir.Symlink("share", install_resource_path / "share"));
+      builder.dir().Symlink("share", install_resource_path / "share"));
 
   // Create the target's `lib` directory.
   std::filesystem::path lib_path =
       std::filesystem::path("lib") / std::string_view(target);
   CARBON_ASSIGN_OR_RETURN(Filesystem::Dir lib_dir,
-                          resource_dir.CreateDirectories(lib_path));
+                          builder.dir().CreateDirectories(lib_path));
 
   llvm::Triple target_triple(target);
   if (target_triple.isOSWindows()) {
@@ -222,15 +227,15 @@ auto ClangRunner::BuildTargetResourceDir(
   // provide the CRT begin/end files, and so we need to build them.
   if (target_triple.isOSLinux()) {
     BuildCrtFile(target, RuntimeSources::CrtBegin,
-                 resource_dir_path / lib_path / "clang_rt.crtbegin.o");
+                 builder.path() / lib_path / "clang_rt.crtbegin.o");
     BuildCrtFile(target, RuntimeSources::CrtEnd,
-                 resource_dir_path / lib_path / "clang_rt.crtend.o");
+                 builder.path() / lib_path / "clang_rt.crtend.o");
   }
 
   CARBON_RETURN_IF_ERROR(
       BuildBuiltinsLib(target, target_triple, tmp_path, lib_dir));
 
-  return Success();
+  return std::move(builder).Commit();
 }
 
 auto ClangRunner::RunInternal(
@@ -508,11 +513,13 @@ auto ClangRunner::BuildBuiltinsLib(llvm::StringRef target,
     objs.push_back(std::move(*obj));
   }
 
-  // Now build an archive out of the `.o` files for the builtins.
+  // Now build an archive out of the `.o` files for the builtins. Note that we
+  // build this directly into the `lib_dir` as this is expected to be a staging
+  // directory and cleaned up on errors.
   std::filesystem::path builtins_a_path = "libclang_rt.builtins.a";
   CARBON_ASSIGN_OR_RETURN(
       Filesystem::WriteFile builtins_a_file,
-      tmp_dir.OpenWriteOnly(builtins_a_path, Filesystem::CreateAlways));
+      lib_dir.OpenWriteOnly(builtins_a_path, Filesystem::CreateAlways));
   {
     llvm::raw_fd_ostream builtins_a_os = builtins_a_file.WriteStream();
 
@@ -528,10 +535,6 @@ auto ClangRunner::BuildBuiltinsLib(llvm::StringRef target,
   }
   CARBON_RETURN_IF_ERROR(std::move(builtins_a_file).Close());
 
-  // Move it into the lib directory.
-  CARBON_RETURN_IF_ERROR(
-      tmp_dir.Rename(builtins_a_path, lib_dir, builtins_a_path));
-
   return Success();
 }