[CAS] Add a mode to load PCMs built from CAS into regular compilation

Usually, the PCMs and PCHs built from CAS are built to be used by other
CAS enabled compilation tasks, thus it has additional steps to
implicitly load dependencies from the CAS.

Add a new option to allow a regular compilation instance to load the
modules built from CAS by ignoring the CAS info inside the serialized
AST. This allows tools like debugger to directly importing the PCMs
without the need to access the CAS. In order to use this option, all
dependencies modules need to be load explicitly.

Author: cachemeifyoucan

clang/include/clang/Driver/Options.td

@@ -7106,6 +7106,11 @@ def fcas_include_tree : Separate<["-"], "fcas-include-tree">,
     HelpText<"Configure the frontend to use a CAS include tree.">,
     MarshallingInfoString<FrontendOpts<"CASIncludeTreeID">>;
 
+defm module_load_ignore_cas : BoolFOption<"module-load-ignore-cas",
+    FrontendOpts<"ModuleLoadIgnoreCAS">, DefaultFalse,
+    PosFlag<SetTrue, [], "Ignore CAS info when loading modules or PCHs">,
+    NegFlag<SetFalse>>;
+
 // FIXME: Add to driver under -fexperimental-cache=compile-job.
 defm cache_compile_job : BoolFOption<"cache-compile-job",
     FrontendOpts<"CacheCompileJob">, DefaultFalse,

clang/include/clang/Frontend/CompilerInstance.h

@@ -758,7 +758,7 @@ class CompilerInstance : public ModuleLoader {
       ArrayRef<std::shared_ptr<DependencyCollector>> DependencyCollectors,
       void *DeserializationListener, bool OwnDeserializationListener,
       bool Preamble, bool UseGlobalModuleIndex,
-      cas::ObjectStore &CAS, cas::ActionCache &Cache,
+      cas::ObjectStore &CAS, cas::ActionCache &Cache, bool ignoreCAS,
       std::unique_ptr<llvm::MemoryBuffer> PCHBuffer = nullptr);
 
   /// Create a code completion consumer using the invocation; note that this

clang/include/clang/Frontend/FrontendOptions.h

@@ -488,6 +488,9 @@ class FrontendOptions {
   /// Use the provided CAS include tree.
   std::string CASIncludeTreeID;
 
+  /// If ignore all the CAS info from serialized AST like modules and PCHs.
+  bool ModuleLoadIgnoreCAS = false;
+
   /// When the input is a module map, the original module map file from which
   /// that map was inferred, if any (for umbrella modules).
   std::string OriginalModuleMap;

clang/lib/Frontend/CompilerInstance.cpp

@@ -676,8 +676,9 @@ void CompilerInstance::createPCHExternalASTSource(
       getASTContext(), getPCHContainerReader(),
       getFrontendOpts().ModuleFileExtensions, DependencyCollectors,
       DeserializationListener, OwnDeserializationListener, Preamble,
-      getFrontendOpts().UseGlobalModuleIndex,
-      getOrCreateObjectStore(), getOrCreateActionCache(), std::move(PCHBuffer));
+      getFrontendOpts().UseGlobalModuleIndex, getOrCreateObjectStore(),
+      getOrCreateActionCache(), getFrontendOpts().ModuleLoadIgnoreCAS,
+      std::move(PCHBuffer));
 }
 
 IntrusiveRefCntPtr<ASTReader> CompilerInstance::createPCHExternalASTSource(
@@ -690,7 +691,7 @@ IntrusiveRefCntPtr<ASTReader> CompilerInstance::createPCHExternalASTSource(
     ArrayRef<std::shared_ptr<DependencyCollector>> DependencyCollectors,
     void *DeserializationListener, bool OwnDeserializationListener,
     bool Preamble, bool UseGlobalModuleIndex,
-    cas::ObjectStore &CAS, cas::ActionCache &Cache,
+    cas::ObjectStore &CAS, cas::ActionCache &Cache, bool ignoreCAS,
     std::unique_ptr<llvm::MemoryBuffer> PCHBuffer) {
   HeaderSearchOptions &HSOpts = PP.getHeaderSearchInfo().getHeaderSearchOpts();
 
@@ -712,8 +713,9 @@ IntrusiveRefCntPtr<ASTReader> CompilerInstance::createPCHExternalASTSource(
   for (auto &Listener : DependencyCollectors)
     Listener->attachToASTReader(*Reader);
 
-  Reader->addListener(std::make_unique<CompileCacheASTReaderHelper>(
-      CAS, Cache, ModuleCache, PP.getDiagnostics()));
+  if (!ignoreCAS)
+    Reader->addListener(std::make_unique<CompileCacheASTReaderHelper>(
+        CAS, Cache, ModuleCache, PP.getDiagnostics()));
 
   auto Listener = std::make_unique<ReadModuleNames>(PP);
   auto &ListenerRef = *Listener;
@@ -1901,9 +1903,10 @@ void CompilerInstance::createASTReader() {
   for (auto &Listener : DependencyCollectors)
     Listener->attachToASTReader(*TheASTReader);
 
-  TheASTReader->addListener(std::make_unique<CompileCacheASTReaderHelper>(
-      getOrCreateObjectStore(), getOrCreateActionCache(), getModuleCache(),
-      getDiagnostics()));
+  if (!FEOpts.ModuleLoadIgnoreCAS)
+    TheASTReader->addListener(std::make_unique<CompileCacheASTReaderHelper>(
+        getOrCreateObjectStore(), getOrCreateActionCache(), getModuleCache(),
+        getDiagnostics()));
 }
 
 bool CompilerInstance::loadModuleFile(StringRef FileName) {

clang/test/CAS/cas-module-file-use-without-cas.c

@@ -0,0 +1,63 @@
+// Tests for reusing a PCM output from CAS builds by a non-cas build.
+// This is to simulate a configuration by debugger without CAS support.
+
+// REQUIRES: ondisk_cas
+
+// RUN: rm -rf %t %t.cas
+// RUN: split-file %s %t
+
+// RUN: llvm-cas --cas %t.cas --ingest %t > %t/casid
+
+// == Build B
+
+// RUN: %clang_cc1 -triple x86_64-apple-macos11 \
+// RUN:   -fmodules -fmodule-name=B -fno-implicit-modules \
+// RUN:   -emit-module %t/module.modulemap -o %t/B.pcm \
+// RUN:   -fcas-path %t.cas -fcas-fs @%t/casid \
+// RUN:   -fcache-compile-job -Rcompile-job-cache &> %t/B.out.txt
+// RUN: cat %t/B.out.txt | sed -E "s:^.*cache [a-z]+ for '([^']+)'.*$:\1:" > %t/B.key
+
+// == Build A, importing B
+
+// RUN: echo -n '-fmodule-file-cache-key %t/B.pcm ' > %t/B.import.rsp
+// RUN: cat %t/B.key >> %t/B.import.rsp
+
+// RUN: %clang_cc1 -triple x86_64-apple-macos11 \
+// RUN:   -fmodules -fmodule-name=A -fno-implicit-modules \
+// RUN:   @%t/B.import.rsp -fmodule-file=%t/B.pcm \
+// RUN:   -emit-module %t/module.modulemap -o %t/A.pcm \
+// RUN:   -fcas-path %t.cas -fcas-fs @%t/casid \
+// RUN:   -fcache-compile-job -Rcompile-job-cache &> %t/A.out.txt
+// RUN: cat %t/A.out.txt | sed -E "s:^.*cache [a-z]+ for '([^']+)'.*$:\1:" > %t/A.key
+
+// == Build tu, importing A and B, without a CAS, this should fail.
+
+// RUN: not %clang_cc1 -triple x86_64-apple-macos11 \
+// RUN:   -fmodules -fno-implicit-modules \
+// RUN:   -fmodule-file=%t/A.pcm\
+// RUN:   -fmodule-file=%t/B.pcm\
+// RUN:   -fsyntax-only %t/tu.c
+
+// == Using option to ignore CAS info inside module
+
+// RUN: %clang_cc1 -triple x86_64-apple-macos11 \
+// RUN:   -fmodules -fno-implicit-modules \
+// RUN:   -fmodule-file=%t/A.pcm\
+// RUN:   -fmodule-file=%t/B.pcm\
+// RUN:   -fsyntax-only %t/tu.c -fmodule-load-ignore-cas
+
+//--- module.modulemap
+module A { header "A.h" export * }
+module B { header "B.h" }
+
+//--- A.h
+#include "B.h"
+
+//--- B.h
+void B(void);
+
+//--- tu.c
+#include "A.h"
+void tu(void) {
+  B();
+}