Merge pull request #81990 from cachemeifyoucan/eng/PR-152441866

[CAS] Allow uncached job from CAS based dependency scanning

Author: cachemeifyoucan

include/swift/Basic/CASOptions.h

@@ -34,6 +34,9 @@ class CASOptions final {
   /// Skip replaying outputs from cache.
   bool CacheSkipReplay = false;
 
+  /// Import modules from CAS.
+  bool ImportModuleFromCAS = false;
+
   /// CASOptions
   clang::CASOptions CASOpts;
 

include/swift/Frontend/Frontend.h

@@ -176,7 +176,8 @@ class CompilerInvocation {
   }
 
   bool requiresCAS() const {
-    return CASOpts.EnableCaching || IRGenOpts.UseCASBackend;
+    return CASOpts.EnableCaching || IRGenOpts.UseCASBackend ||
+           CASOpts.ImportModuleFromCAS;
   }
 
   void setClangModuleCachePath(StringRef Path) {

include/swift/Option/FrontendOptions.td

@@ -244,6 +244,9 @@ def module_can_import_version: MultiArg<["-"], "module-can-import-version", 3>,
   MetaVarName<"<moduleName> <version> <underlyingVersion>">,
   HelpText<"Specify canImport module and versions">;
 
+def module_import_from_cas: Flag<["-"], "module-import-from-cas">,
+  HelpText<"Import modules from CAS instead of file system">;
+
 def disable_cross_import_overlay_search: Flag<["-"], "disable-cross-import-overlay-search">,
   HelpText<"Disable searching for cross import overlay file">;
 

lib/ClangImporter/ClangImporter.cpp

@@ -1161,7 +1161,7 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
     // to reduce the number of argument passing on the command-line and swift
     // compiler can be more efficient to compute swift cache key without having
     // the knowledge about clang command-line options.
-    if (ctx.CASOpts.EnableCaching) {
+    if (ctx.CASOpts.EnableCaching || ctx.CASOpts.ImportModuleFromCAS) {
       CI->getCASOpts() = ctx.CASOpts.CASOpts;
       // When clangImporter is used to compile (generate .pcm or .pch), need to
       // inherit the include tree from swift args (last one wins) and clear the

lib/Frontend/CompilerInvocation.cpp

@@ -800,6 +800,8 @@ static bool ParseCASArgs(CASOptions &Opts, ArgList &Args,
                                             std::string(Value));
   }
 
+  Opts.ImportModuleFromCAS |= Args.hasArg(OPT_module_import_from_cas);
+
   if (auto *A = Args.getLastArg(OPT_clang_include_tree_root))
     Opts.ClangIncludeTree = A->getValue();
   if (auto *A = Args.getLastArg(OPT_clang_include_tree_filelist))

lib/Frontend/Frontend.cpp

@@ -821,7 +821,8 @@ bool CompilerInstance::setUpModuleLoaders() {
   if (ExplicitModuleBuild ||
       !Invocation.getSearchPathOptions().ExplicitSwiftModuleMapPath.empty() ||
       !Invocation.getSearchPathOptions().ExplicitSwiftModuleInputs.empty()) {
-    if (Invocation.getCASOptions().EnableCaching)
+    if (Invocation.getCASOptions().EnableCaching ||
+        Invocation.getCASOptions().ImportModuleFromCAS)
       ESML = ExplicitCASModuleLoader::create(
           *Context, getObjectStore(), getActionCache(), getDependencyTracker(),
           MLM, Invocation.getSearchPathOptions().ExplicitSwiftModuleMapPath,

test/CAS/uncached-casfs.swift

@@ -0,0 +1,79 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O -module-cache-path %t/clang-module-cache \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   -import-objc-header %t/base/Bridging.h -scanner-output-dir %t -auto-bridging-header-chaining -scanner-debug-write-output \
+// RUN:   %t/base/test.swift %t/base/foo.swift -I %t/include -o %t/deps.json -cache-compile-job -cas-path %t/cas
+
+// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/shim.cmd
+// RUN: %swift_frontend_plain @%t/shim.cmd
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:Dummy > %t/dummy.cmd
+// RUN: %swift_frontend_plain @%t/dummy.cmd
+
+// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json bridgingHeader > %t/header.cmd
+// RUN: %target-swift-frontend @%t/header.cmd -disable-implicit-swift-modules -O -o %t/bridging.pch
+// RUN: %cache-tool -cas-path %t/cas -cache-tool-action print-output-keys -- \
+// RUN:   %target-swift-frontend @%t/header.cmd -disable-implicit-swift-modules -O -o %t/bridging.pch > %t/keys.json
+// RUN: %{python} %S/Inputs/ExtractOutputKey.py %t/keys.json > %t/key
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-swift-modules\"" >> %t/MyApp.cmd
+// RUN: echo "\"-import-objc-header\"" >> %t/MyApp.cmd
+// RUN: echo "\"%t/base/Bridging.h\"" >> %t/MyApp.cmd
+// RUN: echo "\"-import-pch\"" >> %t/MyApp.cmd
+// RUN: echo "\"%t/bridging.pch\"" >> %t/MyApp.cmd
+// RUN: echo "\"-bridging-header-pch-key\"" >> %t/MyApp.cmd
+// RUN: echo "\"@%t/key\"" >> %t/MyApp.cmd
+// RUN: echo "\"-explicit-swift-module-map-file\"" >> %t/MyApp.cmd
+// RUN: echo "\"@%t/map.casid\"" >> %t/MyApp.cmd
+
+// RUN: sed -e "s@VFS_DIR@%{/t:regex_replacement}/base@g" -e "s@EXTERNAL_DIR@%{/t:regex_replacement}/modified@g" %t/base.yaml > %t/overlay.yaml
+
+// RUN: %target-swift-frontend %t/base/test.swift %t/base/foo.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \
+// RUN:  -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd -vfsoverlay %t/overlay.yaml -module-import-from-cas
+
+
+//--- base/test.swift
+import Dummy
+public func testFunc() {
+  non_existing_func()
+}
+
+//--- base/foo.swift
+public func foo() {}
+
+//--- modified/test.swift
+import Dummy
+public func testFunc() {
+  dummy()
+  bridge()
+}
+
+//--- base/Bridging.h
+void bridge(void);
+
+//--- include/module.modulemap
+module Dummy {
+ umbrella header "Dummy.h"
+}
+
+//--- include/Dummy.h
+void dummy(void);
+
+//--- base.yaml
+{
+  version: 0,
+  roots: [
+    {
+      type: "directory-remap",
+      name: "VFS_DIR",
+      external-contents: "EXTERNAL_DIR"
+    }
+  ]
+}