[CAS] Move CASConfiguration into LLVM

Move CASConfiguration into LLVM and add a JSON based configuration file
to help constructing CAS from a init file `.cas-config`.

Author: cachemeifyoucan

clang/include/clang/CAS/CASOptions.h

@@ -15,6 +15,7 @@
 #define LLVM_CLANG_CAS_CASOPTIONS_H
 
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/CAS/CASConfiguration.h"
 #include "llvm/Support/Error.h"
 #include <string>
 #include <vector>
@@ -30,52 +31,6 @@ namespace clang {
 
 class DiagnosticsEngine;
 
-/// Base class for options configuring which CAS to use. Separated for the
-/// fields where we don't need special move/copy logic.
-///
-/// TODO: Add appropriate options once we support plugins.
-class CASConfiguration {
-public:
-  enum CASKind {
-    UnknownCAS,
-    InMemoryCAS,
-    OnDiskCAS,
-  };
-
-  /// Kind of CAS to use.
-  CASKind getKind() const {
-    return IsFrozen ? UnknownCAS : CASPath.empty() ? InMemoryCAS : OnDiskCAS;
-  }
-
-  /// Path to a persistent backing store on-disk. This is optional, although \a
-  /// CASFileSystemRootID is unlikely to work without it.
-  ///
-  /// - "" means there is none; falls back to in-memory.
-  /// - "auto" is an alias for an automatically chosen location in the user's
-  ///   system cache.
-  std::string CASPath;
-
-  std::string PluginPath;
-  /// Each entry is a (<option-name>, <value>) pair.
-  std::vector<std::pair<std::string, std::string>> PluginOptions;
-
-  friend bool operator==(const CASConfiguration &LHS,
-                         const CASConfiguration &RHS) {
-    return LHS.CASPath == RHS.CASPath && LHS.PluginPath == RHS.PluginPath &&
-           LHS.PluginOptions == RHS.PluginOptions;
-  }
-  friend bool operator!=(const CASConfiguration &LHS,
-                         const CASConfiguration &RHS) {
-    return !(LHS == RHS);
-  }
-
-private:
-  /// Whether the configuration has been "frozen", in order to hide the kind of
-  /// CAS that's in use.
-  bool IsFrozen = false;
-  friend class CASOptions;
-};
-
 /// Options configuring which CAS to use. User-accessible fields should be
 /// defined in CASConfiguration to enable caching a CAS instance.
 ///
@@ -87,8 +42,18 @@ class CASConfiguration {
 /// clang::createVFSFromCompilerInvocation() uses the same CAS instance that
 /// the rest of the compiler job does, without updating all callers. Probably
 /// it would be better to update all callers and remove it from here.
-class CASOptions : public CASConfiguration {
+class CASOptions : public llvm::cas::CASConfiguration {
 public:
+  enum CASKind {
+    UnknownCAS,
+    InMemoryCAS,
+    OnDiskCAS,
+  };
+
+  /// Kind of CAS to use.
+  CASKind getKind() const {
+    return IsFrozen ? UnknownCAS : CASPath.empty() ? InMemoryCAS : OnDiskCAS;
+  }
   /// Get a CAS & ActionCache defined by the options above. Future calls will
   /// return the same instances... unless the configuration has changed, in
   /// which case new ones will be created.
@@ -117,8 +82,6 @@ class CASOptions : public CASConfiguration {
   /// default on-disk CAS, otherwise this is a noop.
   void ensurePersistentCAS();
 
-  void getResolvedCASPath(llvm::SmallVectorImpl<char> &Result) const;
-
 private:
   /// Initialize Cached CAS and ActionCache.
   llvm::Error initCache() const;
@@ -133,6 +96,10 @@ class CASOptions : public CASConfiguration {
     CASConfiguration Config;
   };
   mutable CachedCAS Cache;
+
+  /// Whether the configuration has been "frozen", in order to hide the kind of
+  /// CAS that's in use.
+  bool IsFrozen = false;
 };
 
 } // end namespace clang

clang/lib/CAS/CASOptions.cpp

@@ -22,7 +22,7 @@ std::pair<std::shared_ptr<llvm::cas::ObjectStore>,
           std::shared_ptr<llvm::cas::ActionCache>>
 CASOptions::getOrCreateDatabases(DiagnosticsEngine &Diags,
                                  bool CreateEmptyDBsOnFailure) const {
-  if (Cache.Config.IsFrozen)
+  if (IsFrozen)
     return {Cache.CAS, Cache.AC};
 
   if (auto E = initCache())
@@ -44,7 +44,7 @@ CASOptions::getOrCreateDatabases() const {
 }
 
 void CASOptions::freezeConfig(DiagnosticsEngine &Diags) {
-  if (Cache.Config.IsFrozen)
+  if (IsFrozen)
     return;
 
   // Make sure the cache is initialized.
@@ -57,7 +57,7 @@ void CASOptions::freezeConfig(DiagnosticsEngine &Diags) {
   // scheduled/executed at a level that has access to the configuration.
   auto &CurrentConfig = static_cast<CASConfiguration &>(*this);
   CurrentConfig = CASConfiguration();
-  CurrentConfig.IsFrozen = Cache.Config.IsFrozen = true;
+  IsFrozen = true;
 
   if (Cache.CAS) {
     // Set the CASPath to the hash schema, since that leaks through CASContext's
@@ -90,41 +90,10 @@ llvm::Error CASOptions::initCache() const {
   Cache.Config = CurrentConfig;
   StringRef CASPath = Cache.Config.CASPath;
 
-  if (!PluginPath.empty()) {
-    std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>> DBs;
-    if (llvm::Error E =
-            createPluginCASDatabases(PluginPath, CASPath, PluginOptions)
-                .moveInto(DBs)) {
-      return E;
-    }
-    std::tie(Cache.CAS, Cache.AC) = std::move(DBs);
-    return llvm::Error::success();
-  }
-
-  if (CASPath.empty()) {
-    Cache.CAS = llvm::cas::createInMemoryCAS();
-    Cache.AC = llvm::cas::createInMemoryActionCache();
-    return llvm::Error::success();
-  }
-
-  SmallString<256> PathBuf;
-  getResolvedCASPath(PathBuf);
-  if (CASPath == "auto") {
-    getDefaultOnDiskCASPath(PathBuf);
-    CASPath = PathBuf;
-  }
-  std::pair<std::unique_ptr<ObjectStore>, std::unique_ptr<ActionCache>> DBs;
-  if (llvm::Error E = createOnDiskUnifiedCASDatabases(CASPath).moveInto(DBs))
-    return E;
+  auto DBs = Cache.Config.createDatabases();
+  if (!DBs)
+    return DBs.takeError();
 
-  std::tie(Cache.CAS, Cache.AC) = std::move(DBs);
+  std::tie(Cache.CAS, Cache.AC) = std::move(*DBs);
   return llvm::Error::success();
 }
-
-void CASOptions::getResolvedCASPath(SmallVectorImpl<char> &Result) const {
-  if (CASPath == "auto") {
-    getDefaultOnDiskCASPath(Result);
-  } else {
-    Result.assign(CASPath.begin(), CASPath.end());
-  }
-}

llvm/include/llvm/CAS/CASConfiguration.h

@@ -0,0 +1,79 @@
+//===- CASOptions.h - Options for configuring the CAS -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Defines the llvm::cas::CASConfiguration interface.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CAS_CASCONFIGURATION_H
+#define LLVM_CAS_CASCONFIGURATION_H
+
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include <string>
+#include <vector>
+
+namespace llvm::cas {
+
+class ActionCache;
+class ObjectStore;
+
+/// Base class for options configuring which CAS to use.
+class CASConfiguration {
+public:
+  /// Path to a persistent backing store on-disk.
+  ///
+  /// - "" means there is none; falls back to in-memory.
+  /// - "auto" is an alias for an automatically chosen location in the user's
+  ///   system cache.
+  std::string CASPath;
+  /// Path to the CAS plugin library.
+  std::string PluginPath;
+  /// Each entry is a (<option-name>, <value>) pair.
+  std::vector<std::pair<std::string, std::string>> PluginOptions;
+
+  friend bool operator==(const CASConfiguration &LHS,
+                         const CASConfiguration &RHS) {
+    return LHS.CASPath == RHS.CASPath && LHS.PluginPath == RHS.PluginPath &&
+           LHS.PluginOptions == RHS.PluginOptions;
+  }
+  friend bool operator!=(const CASConfiguration &LHS,
+                         const CASConfiguration &RHS) {
+    return !(LHS == RHS);
+  }
+
+  // Get resolved CASPath.
+  void getResolvedCASPath(llvm::SmallVectorImpl<char> &Result) const;
+
+  // Create CASDatabase from the CASConfiguration.
+  llvm::Expected<std::pair<std::shared_ptr<llvm::cas::ObjectStore>,
+                           std::shared_ptr<llvm::cas::ActionCache>>>
+  createDatabases() const;
+
+  /// Write CAS configuration file.
+  void writeConfigurationFile(raw_ostream &OS) const;
+
+  /// Create CASConfiguration from config file content.
+  static llvm::Expected<CASConfiguration>
+  createFromConfig(llvm::StringRef Content);
+
+  /// Create CASConfiguration from recurively search config file from a path.
+  ///
+  /// Returns the path to configuration file and its corresponding
+  /// CASConfiguration.
+  static std::optional<std::pair<std::string, CASConfiguration>>
+  createFromSearchConfigFile(
+      StringRef Path,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);
+};
+
+} // namespace llvm::cas
+
+#endif

llvm/lib/CAS/CASConfiguration.cpp

@@ -0,0 +1,122 @@
+//===- CASConfiguration.cpp -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CAS/CASConfiguration.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/JSON.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+
+void CASConfiguration::getResolvedCASPath(
+    llvm::SmallVectorImpl<char> &Result) const {
+  if (CASPath == "auto") {
+    getDefaultOnDiskCASPath(Result);
+  } else {
+    Result.assign(CASPath.begin(), CASPath.end());
+  }
+}
+
+Expected<std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>>>
+CASConfiguration::createDatabases() const {
+  if (!PluginPath.empty())
+    return createPluginCASDatabases(PluginPath, CASPath, PluginOptions);
+
+  if (CASPath.empty()) {
+    return std::pair(createInMemoryCAS(), createInMemoryActionCache());
+  }
+
+  SmallString<128> PathBuf;
+  getResolvedCASPath(PathBuf);
+
+  std::pair<std::unique_ptr<ObjectStore>, std::unique_ptr<ActionCache>> DBs;
+  return createOnDiskUnifiedCASDatabases(PathBuf);
+}
+
+void CASConfiguration::writeConfigurationFile(raw_ostream &OS) const {
+  using namespace llvm::json;
+  Object Root;
+  Root["CASPath"] = CASPath;
+  Root["PluginPath"] = PluginPath;
+
+  Array PlugOpts;
+  for (const auto &Opt : PluginOptions) {
+    Object Entry;
+    Entry[Opt.first] = Opt.second;
+    PlugOpts.emplace_back(std::move(Entry));
+  }
+  Root["PluginOptions"] = std::move(PlugOpts);
+
+  OS << formatv("{0:2}", Value(std::move(Root)));
+}
+
+Expected<CASConfiguration>
+CASConfiguration::createFromConfig(StringRef Content) {
+  auto Parsed = json::parse(Content);
+  if (!Parsed)
+    return Parsed.takeError();
+
+  CASConfiguration Config;
+  auto *Root = Parsed->getAsObject();
+  if (!Root)
+    return createStringError(
+        "CASConfiguration file error: top level object missing");
+
+  if (auto CASPath = Root->getString("CASPath"))
+    Config.CASPath = *CASPath;
+
+  if (auto PluginPath = Root->getString("PluginPath"))
+    Config.PluginPath = *PluginPath;
+
+  if (auto *Opts = Root->getArray("PluginOptions")) {
+    for (auto &Opt : *Opts) {
+      if (auto *Arg = Opt.getAsObject()) {
+        for (auto &Entry : *Arg) {
+          if (auto V = Entry.second.getAsString())
+            Config.PluginOptions.emplace_back(Entry.first.str(), *V);
+        }
+      }
+    }
+  }
+
+  return Config;
+}
+
+std::optional<std::pair<std::string, CASConfiguration>>
+CASConfiguration::createFromSearchConfigFile(
+    StringRef Path, IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
+  if (!VFS)
+    VFS = vfs::getRealFileSystem();
+
+  while (!Path.empty()) {
+    SmallString<256> ConfigPath(Path);
+    sys::path::append(ConfigPath, ".cas-config");
+    auto File = VFS->openFileForRead(ConfigPath);
+    if (!File || !*File) {
+      Path = sys::path::parent_path(Path);
+      continue;
+    }
+
+    auto Buffer = (*File)->getBuffer(ConfigPath);
+    if (!Buffer || !*Buffer) {
+      Path = sys::path::parent_path(Path);
+      continue;
+    }
+
+    auto Config = createFromConfig((*Buffer)->getBuffer());
+    if (!Config) {
+      consumeError(Config.takeError());
+      Path = sys::path::parent_path(Path);
+      continue;
+    }
+    return std::pair{ConfigPath.str().str(), *Config};
+  }
+  return std::nullopt;
+}

llvm/lib/CAS/CMakeLists.txt

@@ -7,6 +7,7 @@ add_llvm_component_library(LLVMCAS
   ActionCaches.cpp
   BuiltinCAS.cpp
   BuiltinUnifiedCASDatabases.cpp
+  CASConfiguration.cpp
   CASFileSystem.cpp
   CASNodeSchema.cpp
   CASOutputBackend.cpp