Merge pull request #6382 from akyrtzi/pr/stable/cas-plugin

[stable][CAS/plugin] Introduce a CAS plugin interface

Author: akyrtzi

clang/include/clang/Basic/DiagnosticCASKinds.td

@@ -11,6 +11,9 @@ let Component = "CAS" in {
 def err_builtin_cas_cannot_be_initialized : Error<
   "CAS cannot be initialized from '%0' on disk (check -fcas-path): %1">,
   DefaultFatal;
+def err_plugin_cas_cannot_be_initialized : Error<
+  "plugin CAS cannot be initialized from '%0': %1">,
+  DefaultFatal;
 def err_builtin_actioncache_cannot_be_initialized : Error<
   "ActionCache cannot be initialized from '%0' on disk (check -fcas-path)">,
   DefaultFatal;

clang/include/clang/CAS/CASOptions.h

@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include <string>
+#include <vector>
 
 namespace llvm {
 namespace cas {
@@ -56,6 +57,10 @@ class CASConfiguration {
   ///   system cache.
   std::string CASPath;
 
+  std::string PluginPath;
+  /// Each entry is a (<option-name>, <value>) pair.
+  std::vector<std::pair<std::string, std::string>> PluginOptions;
+
   friend bool operator==(const CASConfiguration &LHS,
                          const CASConfiguration &RHS) {
     return LHS.CASPath == RHS.CASPath;

clang/include/clang/Driver/Options.td

@@ -6591,6 +6591,16 @@ def fcas_path : Separate<["-"], "fcas-path">,
              " '-fcas-path=auto' chooses a path in the user's system cache.">,
     MarshallingInfoString<CASOpts<"CASPath">>;
 
+def fcas_plugin_path : Separate<["-"], "fcas-plugin-path">,
+    Group<f_Group>, MetaVarName<"<pathname>">,
+    HelpText<"Path to a shared library implementing the LLVM CAS plugin API.">,
+    MarshallingInfoString<CASOpts<"PluginPath">>;
+
+def fcas_plugin_option : Separate<["-"], "fcas-plugin-option">,
+    Group<f_Group>, MetaVarName<"<name>=<value>">,
+    HelpText<"Option to pass to the CAS plugin.">,
+    MarshallingInfoStringPairVector<CASOpts<"PluginOptions">>;
+
 // FIXME: Add to driver once it's supported by -fdepscan.
 def fcas_fs : Separate<["-"], "fcas-fs">,
     Group<f_Group>, MetaVarName<"<tree>">,

clang/lib/CAS/CASOptions.cpp

@@ -59,6 +59,8 @@ void CASOptions::freezeConfig(DiagnosticsEngine &Diags) {
     // API and is observable.
     CurrentConfig.CASPath =
         Cache.CAS->getContext().getHashSchemaIdentifier().str();
+    CurrentConfig.PluginPath.clear();
+    CurrentConfig.PluginOptions.clear();
   }
 }
 
@@ -82,6 +84,20 @@ void CASOptions::initCache(DiagnosticsEngine &Diags) const {
 
   Cache.Config = CurrentConfig;
   StringRef CASPath = Cache.Config.CASPath;
+
+  if (!PluginPath.empty()) {
+    std::pair<std::unique_ptr<ObjectStore>, std::unique_ptr<ActionCache>> DBs;
+    if (llvm::Error E =
+            createPluginCASDatabases(PluginPath, CASPath, PluginOptions)
+                .moveInto(DBs)) {
+      Diags.Report(diag::err_plugin_cas_cannot_be_initialized)
+          << PluginPath << toString(std::move(E));
+      return;
+    }
+    std::tie(Cache.CAS, Cache.AC) = std::move(DBs);
+    return;
+  }
+
   if (CASPath.empty()) {
     Cache.CAS = llvm::cas::createInMemoryCAS();
     Cache.AC = llvm::cas::createInMemoryActionCache();

clang/lib/Frontend/CompilerInvocation.cpp

@@ -403,6 +403,31 @@ static void denormalizeStringVector(SmallVectorImpl<const char *> &Args,
   }
 }
 
+using StringPair = std::pair<std::string, std::string>;
+
+static std::optional<std::vector<std::pair<std::string, std::string>>>
+normalizeStringPairVector(OptSpecifier Opt, int, const ArgList &Args,
+                          DiagnosticsEngine &) {
+  std::vector<std::pair<std::string, std::string>> Pairs;
+  for (StringRef Arg : Args.getAllArgValues(Opt)) {
+    auto [L, R] = Arg.split('=');
+    Pairs.emplace_back(std::string(L), std::string(R));
+  }
+  return Pairs;
+}
+
+static void denormalizeStringPairVector(
+    SmallVectorImpl<const char *> &Args, const char *Spelling,
+    CompilerInvocation::StringAllocator SA, Option::OptionClass OptClass,
+    unsigned TableIndex,
+    const std::vector<std::pair<std::string, std::string>> &Values) {
+  std::vector<std::string> Joined;
+  for (const auto &Pair : Values) {
+    Joined.push_back(Pair.first + "=" + Pair.second);
+  }
+  denormalizeStringVector(Args, Spelling, SA, OptClass, TableIndex, Joined);
+}
+
 static Optional<std::string> normalizeTriple(OptSpecifier Opt, int TableIndex,
                                              const ArgList &Args,
                                              DiagnosticsEngine &Diags) {

clang/test/CAS/driver-cache-launcher.c

@@ -38,6 +38,12 @@
 // SPECIFIC-DAEMON-NOT: "-fdepscan-share-identifier"
 // SPECIFIC-DAEMON: "-cc1depscan" "-fdepscan=daemon" "-fdepscan-daemon=[[PREFIX]]/scand"
 
+// RUN: env LLVM_CACHE_CAS_PATH=%t/cas LLVM_CACHE_PLUGIN_PATH=%t/plugin LLVM_CACHE_PLUGIN_OPTIONS=some-opt=value \
+// RUN:   %clang-cache %clang -c %s -o %t.o -### 2>&1 | FileCheck %s -check-prefix=PLUGIN -DPREFIX=%t
+// PLUGIN: "-fcas-path" "[[PREFIX]]/cas"
+// PLUGIN: "-fcas-plugin-path" "[[PREFIX]]/plugin"
+// PLUGIN: "-fcas-plugin-options" "some-opt=value"
+
 // RUN: env LLVM_CACHE_CAS_PATH=%t/cas LLVM_CACHE_REMOTE_SERVICE_SOCKET_PATH=%t/ccremote %clang-cache %clang -c %s -o %t.o -### 2>&1 | FileCheck %s -check-prefix=REMOTE -DPREFIX=%t
 // REMOTE: "-fcompilation-caching-service-path" "[[PREFIX]]/ccremote"
 

clang/test/CAS/plugin-cas.c

@@ -0,0 +1,30 @@
+// REQUIRES: ondisk_cas
+
+// RUN: rm -rf %t && mkdir -p %t
+
+// RUN: %clang -cc1depscan -o %t/t.rsp -fdepscan=inline -cc1-args \
+// RUN:   -cc1 %s -fcas-path %t/cas \
+// RUN:   -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
+// RUN:   -fcas-plugin-option first-prefix=myfirst- -fcas-plugin-option second-prefix=mysecond-
+// RUN: %clang @%t/t.rsp -emit-obj -o %t/t1.o -Rcompile-job-cache 2>&1 | FileCheck %s --check-prefix=CACHE-MISS
+// RUN: %clang @%t/t.rsp -emit-obj -o %t/t2.o -Rcompile-job-cache 2>&1 | FileCheck %s --check-prefix=CACHE-HIT
+// RUN: diff %t/t1.o %t/t2.o
+
+// CACHE-MISS: remark: compile job cache miss for 'myfirst-mysecond-
+// CACHE-MISS: warning: some warning
+
+// CACHE-HIT: remark: compile job cache hit for 'myfirst-mysecond-
+// CACHE-HIT: warning: some warning
+
+// RUN: not %clang -cc1depscan -o %t/t.rsp -fdepscan=inline -cc1-args \
+// RUN:   -cc1 %s -fcas-path %t/cas \
+// RUN:   -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
+// RUN:   -fcas-plugin-option no-such-option=2 2>&1 | FileCheck %s --check-prefix=FAIL-PLUGIN-OPT
+// FAIL-PLUGIN-OPT: fatal error: plugin CAS cannot be initialized {{.*}}: unknown option: no-such-option
+
+// RUN: not %clang -cc1depscan -o %t/t.rsp -fdepscan=inline -cc1-args \
+// RUN:   -cc1 %s -fcas-path %t/cas -fcas-plugin-path %t/non-existent 2>&1 | FileCheck %s --check-prefix=NOTEXISTENT
+// NOTEXISTENT: fatal error: plugin CAS cannot be initialized
+
+#warning some warning
+void test() {}

clang/test/CMakeLists.txt

@@ -110,6 +110,7 @@ if( NOT CLANG_BUILT_STANDALONE )
   list(APPEND CLANG_TEST_DEPS
     llvm-config
     FileCheck count not
+    CASPluginTest
     llc
     llvm-ar
     llvm-as

clang/tools/driver/CacheLauncherMode.cpp

@@ -91,13 +91,24 @@ static void addCommonArgs(bool ForDriver, SmallVectorImpl<const char *> &Args,
   if (!llvm::sys::Process::GetEnv("CLANG_CACHE_USE_CASFS_DEPSCAN")) {
     Args.push_back("-fdepscan-include-tree");
   }
-  if (auto CASPath = llvm::sys::Process::GetEnv("LLVM_CACHE_CAS_PATH")) {
-    if (ForDriver) {
-      Args.append(
-          {"-Xclang", "-fcas-path", "-Xclang", Saver.save(*CASPath).data()});
-    } else {
-      Args.append({"-fcas-path", Saver.save(*CASPath).data()});
+  auto addCC1Args = [&](ArrayRef<const char *> NewArgs) {
+    for (const char *Arg : NewArgs) {
+      if (ForDriver) {
+        Args.append({"-Xclang", Arg});
+      } else {
+        Args.push_back(Arg);
+      }
     }
+  };
+  if (auto CASPath = llvm::sys::Process::GetEnv("LLVM_CACHE_CAS_PATH")) {
+    addCC1Args({"-fcas-path", Saver.save(*CASPath).data()});
+  }
+  if (auto PluginPath = llvm::sys::Process::GetEnv("LLVM_CACHE_PLUGIN_PATH")) {
+    addCC1Args({"-fcas-plugin-path", Saver.save(*PluginPath).data()});
+  }
+  if (auto PluginOpts =
+          llvm::sys::Process::GetEnv("LLVM_CACHE_PLUGIN_OPTIONS")) {
+    addCC1Args({"-fcas-plugin-options", Saver.save(*PluginOpts).data()});
   }
 }