[clang][cas] Adopt validate-if-needed in clang scanning daemon

Use the new validate-if-needed functionality to ensure the clang
scanning daemon's CAS data is valid.

(cherry picked from commit 1101d73)

Author: benlangmuir

clang/include/clang/CAS/CASOptions.h

@@ -117,6 +117,8 @@ class CASOptions : public CASConfiguration {
   /// default on-disk CAS, otherwise this is a noop.
   void ensurePersistentCAS();
 
+  std::string getResolvedCASPath() const;
+
 private:
   /// Initialize Cached CAS and ActionCache.
   llvm::Error initCache() const;

clang/lib/CAS/CASOptions.cpp

@@ -119,3 +119,12 @@ llvm::Error CASOptions::initCache() const {
   std::tie(Cache.CAS, Cache.AC) = std::move(DBs);
   return llvm::Error::success();
 }
+
+std::string CASOptions::getResolvedCASPath() const {
+  if (CASPath != "auto")
+    return CASPath;
+
+  SmallString<256> PathBuf;
+  getDefaultOnDiskCASPath(PathBuf);
+  return std::string(PathBuf);
+}

clang/test/CAS/depscan-cas-log.c

@@ -3,7 +3,7 @@
 // enable logging there are currently zero records in the log.
 
 // RUN: rm -rf %t && mkdir %t
-// RUN: env LLVM_CACHE_CAS_PATH=%t/cas LLVM_CAS_LOG=1 %clang \
+// RUN: env LLVM_CACHE_CAS_PATH=%t/cas LLVM_CAS_LOG=1 LLVM_CAS_DISABLE_VALIDATION=1 %clang \
 // RUN:   -cc1depscan -fdepscan=daemon -fdepscan-include-tree -o - \
 // RUN:   -cc1-args -cc1 -triple x86_64-apple-macosx11.0.0 -emit-obj %s -o %t/t.o -fcas-path %t/cas
 // RUN: FileCheck %s --input-file %t/cas/v1.log

clang/test/CAS/validate-once.c

@@ -0,0 +1,18 @@
+// RUN: rm -rf %t
+
+// RUN: llvm-cas --cas %t/cas --ingest %s
+// RUN: mv %t/cas/v1.1/v8.data %t/cas/v1.1/v8.data.bak
+
+// RUN: %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t -cas-args -fcas-path %t/cas -- \
+// RUN:   %clang -target x86_64-apple-macos11 -I %S/Inputs \
+// RUN:     -Xclang -fcas-path -Xclang %t/cas \
+// RUN:     -fdepscan=daemon -fdepscan-daemon=%{clang-daemon-dir}/%basename_t -fsyntax-only -x c %s
+
+// RUN: ls %t/cas/corrupt.0.v1.1
+
+// RUN: llvm-cas --cas %t/cas --validate-if-needed | FileCheck %s -check-prefix=SKIPPED
+// SKIPPED: validation skipped
+
+#include "test.h"
+
+int func(void);

clang/tools/driver/cc1depscanProtocol.cpp

@@ -188,7 +188,8 @@ Expected<ScanDaemon> ScanDaemon::launchDaemon(StringRef BasePath,
 #endif
 
   static constexpr const char *PassThroughEnv[] = {
-    "LLVM_CAS_LOG",
+      "LLVM_CAS_LOG",
+      "LLVM_CAS_DISABLE_VALIDATION",
   };
   SmallVector<const char *> EnvP;
   for (const char *Name : PassThroughEnv)

clang/tools/driver/cc1depscan_main.cpp

@@ -28,9 +28,11 @@
 #include "clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Bitstream/BitstreamReader.h"
 #include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
 #include "llvm/CAS/CASProvidingFileSystem.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
 #include "llvm/CAS/HierarchicalTreeBuilder.h"
@@ -41,6 +43,7 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PrefixMapper.h"
@@ -52,6 +55,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include <cstdio>
 #include <mutex>
+#include <optional>
 #include <shared_mutex>
 
 #if LLVM_ON_UNIX
@@ -631,8 +635,8 @@ namespace {
 struct ScanServer {
   const char *Argv0 = nullptr;
   SmallString<128> BasePath;
-  /// List of cas options.
-  ArrayRef<const char *> CASArgs;
+  CASOptions CASOpts;
+  bool ProduceIncludeTree = true;
   int PidFD = -1;
   int ListenSocket = -1;
   /// \p std::nullopt means it runs indefinitely.
@@ -641,7 +645,7 @@ struct ScanServer {
 
   ~ScanServer() { shutdown(); }
 
-  void start(bool Exclusive);
+  void start(bool Exclusive, ArrayRef<const char *> CASArgs);
   int listen();
 
   /// Tear down the socket and bind file immediately but wait till all existing
@@ -706,13 +710,13 @@ int cc1depscand_main(ArrayRef<const char *> Argv, const char *Argv0,
   // particular "build session", to shutdown, then have it stay alive until the
   // session is finished.
   bool LongRunning = false;
-
+  ArrayRef<const char *> CASArgs;
   for (const auto *A = Argv.begin() + 2; A != Argv.end(); ++A) {
     StringRef Arg(*A);
     if (Arg == "-long-running")
       LongRunning = true;
     else if (Arg == "-cas-args") {
-      Server.CASArgs = ArrayRef(A + 1, Argv.end());
+      CASArgs = ArrayRef(A + 1, Argv.end());
       break;
     }
   }
@@ -723,7 +727,7 @@ int cc1depscand_main(ArrayRef<const char *> Argv, const char *Argv0,
     reportError(Twine("cannot create basedir: ") + EC.message());
 
   if (Command == "-serve") {
-    Server.start(/*Exclusive*/ true);
+    Server.start(/*Exclusive*/ true, CASArgs);
     return Server.listen();
 
   } else if (Command == "-execute") {
@@ -734,7 +738,7 @@ int cc1depscand_main(ArrayRef<const char *> Argv, const char *Argv0,
     }
 
     // Make sure to start the server before executing the command.
-    Server.start(/*Exclusive*/ true);
+    Server.start(/*Exclusive*/ true, CASArgs);
     std::thread ServerThread([&Server]() { Server.listen(); });
 
     setenv("CLANG_CACHE_SCAN_DAEMON_SOCKET_PATH", Server.BasePath.c_str(),
@@ -785,11 +789,59 @@ int cc1depscand_main(ArrayRef<const char *> Argv, const char *Argv0,
   openAndReplaceFD(1, LogOutPath);
   openAndReplaceFD(2, LogErrPath);
 
-  Server.start(/*Exclusive*/ false);
+  Server.start(/*Exclusive*/ false, CASArgs);
   return Server.listen();
 }
 
-void ScanServer::start(bool Exclusive) {
+static std::optional<StringRef>
+findLLVMCasBinary(const char *Argv0, llvm::SmallVectorImpl<char> &Storage) {
+  using namespace llvm::sys;
+  std::string Path = fs::getMainExecutable(Argv0, (void *)cc1depscan_main);
+  Storage.assign(Path.begin(), Path.end());
+  path::remove_filename(Storage);
+  path::append(Storage, "llvm-cas");
+  StringRef PathStr(Storage.data(), Storage.size());
+  if (fs::exists(PathStr))
+    return PathStr;
+  // Look for a corresponding usr/local/bin/llvm-cas
+  PathStr = path::parent_path(PathStr);
+  if (path::filename(PathStr) != "bin")
+    return std::nullopt;
+  PathStr = path::parent_path(PathStr);
+  Storage.truncate(PathStr.size());
+  path::append(Storage, "local", "bin", "llvm-cas");
+  PathStr = StringRef{Storage.data(), Storage.size()};
+  if (fs::exists(PathStr))
+    return PathStr;
+  return std::nullopt;
+}
+
+void ScanServer::start(bool Exclusive, ArrayRef<const char *> CASArgs) {
+  // Parse CAS options and validate if needed.
+  DiagnosticsEngine Diags(new DiagnosticIDs(), new DiagnosticOptions());
+
+  const OptTable &Opts = clang::driver::getDriverOptTable();
+  unsigned MissingArgIndex, MissingArgCount;
+  auto ParsedCASArgs =
+      Opts.ParseArgs(CASArgs, MissingArgIndex, MissingArgCount);
+  CompilerInvocation::ParseCASArgs(CASOpts, ParsedCASArgs, Diags);
+  CASOpts.ensurePersistentCAS();
+  ProduceIncludeTree =
+      ParsedCASArgs.hasArg(driver::options::OPT_fdepscan_include_tree);
+
+  static std::once_flag ValidateOnce;
+  std::call_once(ValidateOnce, [&] {
+    if (getenv("LLVM_CAS_DISABLE_VALIDATION"))
+      return;
+    if (CASOpts.CASPath.empty() || !CASOpts.PluginPath.empty())
+      return;
+    SmallString<64> LLVMCasStorage;
+    ExitOnErr(llvm::cas::validateOnDiskUnifiedCASDatabasesIfNeeded(
+        CASOpts.getResolvedCASPath(), /*CheckHash=*/true,
+        /*AllowRecovery=*/true,
+        /*Force=*/false, findLLVMCasBinary(Argv0, LLVMCasStorage)));
+  });
+
   // Check the pidfile.
   SmallString<128> PidPath;
   (BasePath + ".pid").toVector(PidPath);
@@ -828,16 +880,6 @@ int ScanServer::listen() {
   llvm::DefaultThreadPool Pool;
 
   DiagnosticsEngine Diags(new DiagnosticIDs(), new DiagnosticOptions());
-  CASOptions CASOpts;
-  const OptTable &Opts = clang::driver::getDriverOptTable();
-  unsigned MissingArgIndex, MissingArgCount;
-  auto ParsedCASArgs =
-      Opts.ParseArgs(CASArgs, MissingArgIndex, MissingArgCount);
-  CompilerInvocation::ParseCASArgs(CASOpts, ParsedCASArgs, Diags);
-  CASOpts.ensurePersistentCAS();
-  bool ProduceIncludeTree =
-      ParsedCASArgs.hasArg(driver::options::OPT_fdepscan_include_tree);
-
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
   std::shared_ptr<llvm::cas::ActionCache> Cache;
   std::tie(CAS, Cache) = CASOpts.getOrCreateDatabases(Diags);