Merge pull request #6337 from akyrtzi/pr/stable/ondisk-cas-revamp

[stable] Cherry-pick commits for on-disk CAS revamp

Author: akyrtzi

clang/include/clang/Basic/DiagnosticCASKinds.td

@@ -9,10 +9,10 @@
 let Component = "CAS" in {
 
 def err_builtin_cas_cannot_be_initialized : Error<
-  "CAS cannot be initialized from '%0' on disk (check -fcas-path)">,
+  "CAS cannot be initialized from '%0' on disk (check -fcas-path): %1">,
   DefaultFatal;
 def err_builtin_actioncache_cannot_be_initialized : Error<
-  "ActionCache cannot be initialized from '%0' on disk (check -faction-cache-path)">,
+  "ActionCache cannot be initialized from '%0' on disk (check -fcas-path)">,
   DefaultFatal;
 def err_cas_cannot_parse_root_id : Error<
   "CAS cannot parse root-id '%0' specified by -fcas-fs">, DefaultFatal;

clang/include/clang/CAS/CASOptions.h

@@ -14,12 +14,10 @@
 #ifndef LLVM_CLANG_CAS_CASOPTIONS_H
 #define LLVM_CLANG_CAS_CASOPTIONS_H
 
-#include <memory>
+#include "llvm/ADT/SmallVector.h"
 #include <string>
-#include <vector>
 
 namespace llvm {
-class Error;
 namespace cas {
 class ActionCache;
 class ObjectStore;
@@ -30,6 +28,9 @@ namespace clang {
 
 class DiagnosticsEngine;
 
+/// Set \p Path to a reasonable default on-disk cache path for the current user.
+void getClangDefaultCachePath(llvm::SmallVectorImpl<char> &Path);
+
 /// Base class for options configuring which CAS to use. Separated for the
 /// fields where we don't need special move/copy logic.
 ///
@@ -54,11 +55,10 @@ class CASConfiguration {
   /// - "auto" is an alias for an automatically chosen location in the user's
   ///   system cache.
   std::string CASPath;
-  std::string CachePath;
 
   friend bool operator==(const CASConfiguration &LHS,
                          const CASConfiguration &RHS) {
-    return LHS.CASPath == RHS.CASPath && LHS.CachePath == RHS.CachePath;
+    return LHS.CASPath == RHS.CASPath;
   }
   friend bool operator!=(const CASConfiguration &LHS,
                          const CASConfiguration &RHS) {
@@ -75,8 +75,8 @@ class CASConfiguration {
 /// Options configuring which CAS to use. User-accessible fields should be
 /// defined in CASConfiguration to enable caching a CAS instance.
 ///
-/// CASOptions includes \a getOrCreateObjectStore() and \a
-/// getOrCreateActionCache() for creating CAS and ActionCache.
+/// CASOptions includes \a getOrCreateDatabases() for creating CAS and
+/// ActionCache.
 ///
 /// FIXME: The the caching is done here, instead of as a field in \a
 /// CompilerInstance, in order to ensure that \a
@@ -85,32 +85,23 @@ class CASConfiguration {
 /// it would be better to update all callers and remove it from here.
 class CASOptions : public CASConfiguration {
 public:
-  /// Get a CAS defined by the options above. Future calls will return the same
-  /// CAS instance... unless the configuration has changed, in which case a new
-  /// one will be created.
-  ///
-  /// If \p CreateEmptyCASOnFailure, returns an empty in-memory CAS on failure.
-  /// Else, returns \c nullptr on failure.
-  std::shared_ptr<llvm::cas::ObjectStore>
-  getOrCreateObjectStore(DiagnosticsEngine &Diags,
-                         bool CreateEmptyCASOnFailure = false) const;
-
-  /// Get a ActionCache defined by the options above. Future calls will return
-  /// the same ActionCache instance... unless the configuration has changed, in
-  /// which case a new one will be created.
+  /// Get a CAS & ActionCache defined by the options above. Future calls will
+  /// return the same instances... unless the configuration has changed, in
+  /// which case new ones will be created.
   ///
-  /// If \p CreateEmptyCacheOnFailure, returns an empty in-memory ActionCache on
+  /// If \p CreateEmptyDBsOnFailure, returns empty in-memory databases on
   /// failure. Else, returns \c nullptr on failure.
-  std::shared_ptr<llvm::cas::ActionCache>
-  getOrCreateActionCache(DiagnosticsEngine &Diags,
-                         bool CreateEmptyCacheOnFailures = false) const;
+  std::pair<std::shared_ptr<llvm::cas::ObjectStore>,
+            std::shared_ptr<llvm::cas::ActionCache>>
+  getOrCreateDatabases(DiagnosticsEngine &Diags,
+                       bool CreateEmptyDBsOnFailure = false) const;
 
   /// Freeze CAS Configuration. Future calls will return the same
   /// CAS instance, even if the configuration changes again later.
   ///
   /// The configuration will be wiped out to prevent it being observable or
   /// affecting the output of something that takes \a CASOptions as an input.
-  /// This also "locks in" the return value of \a getOrCreateObjectStore():
+  /// This also "locks in" the return value of \a getOrCreateDatabases():
   /// future calls will not check if the configuration has changed.
   void freezeConfig(DiagnosticsEngine &Diags);
 

clang/include/clang/Driver/Options.td

@@ -6591,12 +6591,6 @@ def fcas_path : Separate<["-"], "fcas-path">,
              " '-fcas-path=auto' chooses a path in the user's system cache.">,
     MarshallingInfoString<CASOpts<"CASPath">>;
 
-def faction_cache_path : Separate<["-"], "faction-cache-path">,
-    Group<f_Group>, MetaVarName<"<dir>|auto">,
-    HelpText<"Path to a persistent on-disk backing store for the builtin Cache."
-             " '-faction-cache-path=auto' chooses a path in the user's system cache.">,
-    MarshallingInfoString<CASOpts<"CachePath">>;
-
 // FIXME: Add to driver once it's supported by -fdepscan.
 def fcas_fs : Separate<["-"], "fcas-fs">,
     Group<f_Group>, MetaVarName<"<tree>">,

clang/include/clang/Frontend/CompilerInstance.h

@@ -186,6 +186,8 @@ class CompilerInstance : public ModuleLoader {
   /// Force an output buffer.
   std::unique_ptr<llvm::raw_pwrite_stream> OutputStream;
 
+  void createCASDatabases();
+
   CompilerInstance(const CompilerInstance &) = delete;
   void operator=(const CompilerInstance &) = delete;
 public:

clang/lib/CAS/CASOptions.cpp

@@ -10,46 +10,33 @@
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticCAS.h"
 #include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
 #include "llvm/CAS/ObjectStore.h"
-#include "llvm/Support/Error.h"
+#include "llvm/Support/Path.h"
 
 using namespace clang;
 using namespace llvm::cas;
 
-static std::shared_ptr<llvm::cas::ObjectStore>
-createObjectStore(const CASConfiguration &Config, DiagnosticsEngine &Diags) {
-  if (Config.CASPath.empty())
-    return llvm::cas::createInMemoryCAS();
-
-  // Compute the path.
-  SmallString<128> Storage;
-  StringRef Path = Config.CASPath;
-  if (Path == "auto") {
-    llvm::cas::getDefaultOnDiskCASPath(Storage);
-    Path = Storage;
-  }
-
-  // FIXME: Pass on the actual error from the CAS.
-  if (auto MaybeCAS =
-          llvm::expectedToOptional(llvm::cas::createOnDiskCAS(Path)))
-    return std::move(*MaybeCAS);
-  Diags.Report(diag::err_builtin_cas_cannot_be_initialized) << Path;
-  return nullptr;
+void clang::getClangDefaultCachePath(SmallVectorImpl<char> &Path) {
+  // FIXME: Should this return 'Error' instead of hard-failing?
+  if (!llvm::sys::path::cache_directory(Path))
+    llvm::report_fatal_error("cannot get default cache directory");
+  llvm::sys::path::append(Path, "clang-cache");
 }
 
-std::shared_ptr<llvm::cas::ObjectStore>
-CASOptions::getOrCreateObjectStore(DiagnosticsEngine &Diags,
-                                   bool CreateEmptyCASOnFailure) const {
+std::pair<std::shared_ptr<llvm::cas::ObjectStore>,
+          std::shared_ptr<llvm::cas::ActionCache>>
+CASOptions::getOrCreateDatabases(DiagnosticsEngine &Diags,
+                                 bool CreateEmptyDBsOnFailure) const {
   if (Cache.Config.IsFrozen)
-    return Cache.CAS;
+    return {Cache.CAS, Cache.AC};
 
   initCache(Diags);
-  if (Cache.CAS)
-    return Cache.CAS;
-  if (!CreateEmptyCASOnFailure)
-    return nullptr;
-  Cache.CAS = llvm::cas::createInMemoryCAS();
-  return Cache.CAS;
+  if (!Cache.CAS && CreateEmptyDBsOnFailure)
+    Cache.CAS = llvm::cas::createInMemoryCAS();
+  if (!Cache.AC && CreateEmptyDBsOnFailure)
+    Cache.AC = llvm::cas::createInMemoryActionCache();
+  return {Cache.CAS, Cache.AC};
 }
 
 void CASOptions::freezeConfig(DiagnosticsEngine &Diags) {
@@ -73,53 +60,15 @@ void CASOptions::freezeConfig(DiagnosticsEngine &Diags) {
     CurrentConfig.CASPath =
         Cache.CAS->getContext().getHashSchemaIdentifier().str();
   }
-  if (Cache.AC)
-    CurrentConfig.CachePath = "";
-}
-
-static std::shared_ptr<llvm::cas::ActionCache>
-createCache(ObjectStore &CAS, const CASConfiguration &Config,
-            DiagnosticsEngine &Diags) {
-  if (Config.CachePath.empty())
-    return llvm::cas::createInMemoryActionCache();
-
-  // Compute the path.
-  std::string Path = Config.CachePath;
-  if (Path == "auto")
-    Path = getDefaultOnDiskActionCachePath();
-
-  // FIXME: Pass on the actual error from the CAS.
-  if (auto MaybeCache =
-          llvm::expectedToOptional(llvm::cas::createOnDiskActionCache(Path)))
-    return std::move(*MaybeCache);
-  Diags.Report(diag::err_builtin_actioncache_cannot_be_initialized) << Path;
-  return nullptr;
-}
-
-std::shared_ptr<llvm::cas::ActionCache>
-CASOptions::getOrCreateActionCache(DiagnosticsEngine &Diags,
-                                   bool CreateEmptyOnFailure) const {
-  if (Cache.Config.IsFrozen)
-    return Cache.AC;
-
-  initCache(Diags);
-  if (Cache.AC)
-    return Cache.AC;
-  if (!CreateEmptyOnFailure)
-    return nullptr;
-
-  Cache.CAS = Cache.CAS ? Cache.CAS : llvm::cas::createInMemoryCAS();
-  return llvm::cas::createInMemoryActionCache();
 }
 
 void CASOptions::ensurePersistentCAS() {
   assert(!IsFrozen && "Expected to check for a persistent CAS before freezing");
   switch (getKind()) {
   case UnknownCAS:
-      llvm_unreachable("Cannot ensure persistent CAS if it's unknown / frozen");
+    llvm_unreachable("Cannot ensure persistent CAS if it's unknown / frozen");
   case InMemoryCAS:
     CASPath = "auto";
-    CachePath = "auto";
     break;
   case OnDiskCAS:
     break;
@@ -132,6 +81,23 @@ void CASOptions::initCache(DiagnosticsEngine &Diags) const {
     return;
 
   Cache.Config = CurrentConfig;
-  Cache.CAS = createObjectStore(Cache.Config, Diags);
-  Cache.AC = createCache(*Cache.CAS, Cache.Config, Diags);
+  StringRef CASPath = Cache.Config.CASPath;
+  if (CASPath.empty()) {
+    Cache.CAS = llvm::cas::createInMemoryCAS();
+    Cache.AC = llvm::cas::createInMemoryActionCache();
+    return;
+  }
+
+  SmallString<256> PathBuf;
+  if (CASPath == "auto") {
+    getClangDefaultCachePath(PathBuf);
+    CASPath = PathBuf;
+  }
+  std::pair<std::unique_ptr<ObjectStore>, std::unique_ptr<ActionCache>> DBs;
+  if (llvm::Error E = createOnDiskUnifiedCASDatabases(CASPath).moveInto(DBs)) {
+    Diags.Report(diag::err_builtin_cas_cannot_be_initialized)
+        << CASPath << toString(std::move(E));
+    return;
+  }
+  std::tie(Cache.CAS, Cache.AC) = std::move(DBs);
 }

clang/lib/Frontend/CompilerInstance.cpp

@@ -902,25 +902,24 @@ llvm::vfs::OutputBackend &CompilerInstance::getOrCreateOutputBackend() {
   return getOutputBackend();
 }
 
-llvm::cas::ObjectStore &CompilerInstance::getOrCreateObjectStore() {
-  if (CAS)
-    return *CAS;
-
-  // Create a new CAS instance from the CompilerInvocation. Future calls to
+void CompilerInstance::createCASDatabases() {
+  // Create a new CAS databases from the CompilerInvocation. Future calls to
   // createFileManager() will use the same CAS.
-  CAS = getInvocation().getCASOpts().getOrCreateObjectStore(
-      getDiagnostics(),
-      /*CreateEmptyCASOnFailure=*/true);
+  std::tie(CAS, ActionCache) =
+      getInvocation().getCASOpts().getOrCreateDatabases(
+          getDiagnostics(),
+          /*CreateEmptyCASOnFailure=*/true);
+}
+
+llvm::cas::ObjectStore &CompilerInstance::getOrCreateObjectStore() {
+  if (!CAS)
+    createCASDatabases();
   return *CAS;
 }
 
 llvm::cas::ActionCache &CompilerInstance::getOrCreateActionCache() {
-  if (ActionCache)
-    return *ActionCache;
-
-  ActionCache = getInvocation().getCASOpts().getOrCreateActionCache(
-      getDiagnostics(),
-      /*CreateEmptyActionCacheOnFailure=*/true);
+  if (!ActionCache)
+    createCASDatabases();
   return *ActionCache;
 }
 
@@ -2445,4 +2444,4 @@ bool CompileCacheASTReaderHelper::readCASFileSystemRootID(StringRef RootID,
     return true;
   }
   return false;
-}
+}

clang/lib/Frontend/CompilerInvocation.cpp

@@ -1347,7 +1347,7 @@ createBaseFS(const FileSystemOptions &FSOpts, const FrontendOptions &FEOpts,
   // If no CAS was provided, create one with CASOptions.
   std::shared_ptr<llvm::cas::ObjectStore> CAS = std::move(OverrideCAS);
   if (!CAS)
-    CAS = CASOpts.getOrCreateObjectStore(Diags);
+    CAS = CASOpts.getOrCreateDatabases(Diags).first;
 
   // Helper for creating a valid (but empty) CASFS if an error is encountered.
   auto makeEmptyCASFS = [&CAS]() {
@@ -2988,7 +2988,7 @@ static void determineInputFromIncludeTree(
     Diags.Report(diag::err_fe_unable_to_load_include_tree)
         << IncludeTreeID << llvm::toString(std::move(E));
   };
-  auto CAS = CASOpts.getOrCreateObjectStore(Diags);
+  auto CAS = CASOpts.getOrCreateDatabases(Diags).first;
   if (!CAS)
     return;
   auto ID = CAS->parseID(IncludeTreeID);

clang/test/CAS/cached-diagnostics.c

@@ -5,15 +5,15 @@
 // RUN: %clang_cc1 -triple x86_64-apple-macos12 -fsyntax-only %t/src/main.c -I %t/src/inc -Wunknown-pragmas 2> %t/regular-diags1.txt
 
 // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -fdepscan-prefix-map=%t/src=/^src -o %t/t1.rsp -cc1-args \
-// RUN:   -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas -faction-cache-path %t/cache \
+// RUN:   -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas \
 // RUN:     -emit-obj %t/src/main.c -o %t/out/output.o -I %t/src/inc -Wunknown-pragmas
 
 // Compare diagnostics after a miss.
 // RUN: %clang @%t/t1.rsp 2> %t/diags1.txt
 // RUN: diff -u %t/regular-diags1.txt %t/diags1.txt
 
 // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t/t1.noprefix.rsp -cc1-args \
-// RUN:   -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas -faction-cache-path %t/cache \
+// RUN:   -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas \
 // RUN:     -emit-obj %t/src/main.c -o %t/out/output.o -I %t/src/inc -Wunknown-pragmas
 
 // Compare diagnostics without prefix mappings.
@@ -42,7 +42,7 @@
 // RUN: %clang_cc1 -triple x86_64-apple-macos12 -fsyntax-only %t/src2/main.c -I %t/src2/inc -Wunknown-pragmas 2> %t/regular-diags2.txt
 
 // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -fdepscan-prefix-map=%t/src2=/^src -o %t/t2.rsp -cc1-args \
-// RUN:   -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas -faction-cache-path %t/cache \
+// RUN:   -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas \
 // RUN:     -emit-obj %t/src2/main.c -o %t/out2/output.o -I %t/src2/inc -Wunknown-pragmas
 // RUN: %clang @%t/t2.rsp -Rcompile-job-cache 2> %t/diags-hit2.txt
 

clang/test/CAS/daemon-cwd.c

@@ -11,13 +11,13 @@
 // RUN:    -fdepscan-prefix-map=%t=/^build                                \
 // RUN:    -fdepscan-prefix-map-toolchain=/^toolchain                     \
 // RUN:    -fdepscan-daemon=%{clang-daemon-dir}/%basename_t               \
-// RUN:    -Xclang -fcas-path -Xclang %t/cas -Xclang -faction-cache-path -Xclang %t/cache \
+// RUN:    -Xclang -fcas-path -Xclang %t/cas                              \
 // RUN:    -MD -MF %t/test.d -Iinclude                                    \
 // RUN:    -fsyntax-only -x c %s >> %t/cmd.sh
 // RUN: chmod +x %t/cmd.sh
 
 // RUN: %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t      \
-// RUN:   -cas-args -fcas-path %t/cas -faction-cache-path %t/cache -- %t/cmd.sh
+// RUN:   -cas-args -fcas-path %t/cas -- %t/cmd.sh
 // RUN: (cd %t && %clang -target x86_64-apple-macos11 -MD -MF %t/test2.d  \
 // RUN:    -Iinclude -fsyntax-only -x c %s)
 // RUN: diff %t/test.d %t/test2.d

clang/test/CAS/depscan-dependency-file.c

@@ -2,14 +2,14 @@
 // RUN: split-file %s %t
 
 // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t/t.rsp -cc1-args \
-// RUN:   -cc1 -fcas-path %t/cas -faction-cache-path %t/cache -triple x86_64-apple-macos11 %t/main.c -emit-obj -o %t/output.o -isystem %t/sys \
+// RUN:   -cc1 -fcas-path %t/cas -triple x86_64-apple-macos11 %t/main.c -emit-obj -o %t/output.o -isystem %t/sys \
 // RUN:     -MT deps -dependency-file %t/t.d
 // RUN: FileCheck %s -input-file=%t/t.d -check-prefix=NOSYS
 // RUN: FileCheck %s -input-file=%t/t.d -check-prefix=COMMON
 
 // Including system headers.
 // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t/t.rsp -cc1-args \
-// RUN:   -cc1 -fcas-path %t/cas -faction-cache-path %t/cache -triple x86_64-apple-macos11 %t/main.c -emit-obj -o %t/output.o -isystem %t/sys \
+// RUN:   -cc1 -fcas-path %t/cas -triple x86_64-apple-macos11 %t/main.c -emit-obj -o %t/output.o -isystem %t/sys \
 // RUN:     -MT deps -sys-header-deps -dependency-file %t/t-sys.d
 // RUN: FileCheck %s -input-file=%t/t-sys.d -check-prefix=WITHSYS -check-prefix=COMMON