Merge pull request #78282 from cachemeifyoucan/eng/PR-141567785-release

[6.1][Caching] Fix the cache replay bugs from driver C APIs

Author: cachemeifyoucan

include/swift/AST/DiagnosticsFrontend.def

@@ -511,6 +511,7 @@ REMARK(output_cache_miss, none, "cache miss for input file '%0': key '%1'", (Str
 
 // CAS related diagnostics
 ERROR(error_cas, none, "CAS error encountered: %0", (StringRef))
+WARNING(cache_replay_failed, none, "cache replay failed: %0", (StringRef))
 
 ERROR(error_failed_cached_diag, none, "failed to serialize cached diagnostics: %0", (StringRef))
 ERROR(error_replay_cached_diag, none, "failed to replay cached diagnostics: %0", (StringRef))

include/swift/AST/IRGenOptions.h

@@ -306,7 +306,7 @@ class IRGenOptions {
 
   /// Whether we're generating IR for the JIT.
   unsigned UseJIT : 1;
-  
+
   /// Whether we should run LLVM optimizations after IRGen.
   unsigned DisableLLVMOptzns : 1;
 
@@ -377,7 +377,7 @@ class IRGenOptions {
   /// Force public linkage for private symbols. Used only by the LLDB
   /// expression evaluator.
   unsigned ForcePublicLinkage : 1;
-  
+
   /// Force lazy initialization of class metadata
   /// Used on Windows to avoid cross-module references.
   unsigned LazyInitializeClassMetadata : 1;
@@ -439,7 +439,7 @@ class IRGenOptions {
   /// Whether to disable shadow copies for local variables on the stack. This is
   /// only used for testing.
   unsigned DisableDebuggerShadowCopies : 1;
-  
+
   /// Whether to disable using mangled names for accessing concrete type metadata.
   unsigned DisableConcreteTypeMetadataMangledNameAccessors : 1;
 
@@ -524,7 +524,7 @@ class IRGenOptions {
   };
 
   TypeInfoDumpFilter TypeInfoFilter;
-  
+
   /// Pull in runtime compatibility shim libraries by autolinking.
   std::optional<llvm::VersionTuple> AutolinkRuntimeCompatibilityLibraryVersion;
   std::optional<llvm::VersionTuple>
@@ -543,13 +543,13 @@ class IRGenOptions {
   llvm::CallingConv::ID PlatformCCallingConvention;
 
   /// Use CAS based object format as the output.
-  bool UseCASBackend;
+  bool UseCASBackend = false;
 
   /// The output mode for the CAS Backend.
   llvm::CASBackendMode CASObjMode;
 
   /// Emit a .casid file next to the object file if CAS Backend is used.
-  bool EmitCASIDFile;
+  bool EmitCASIDFile = false;
 
   IRGenOptions()
       : OutputKind(IRGenOutputKind::LLVMAssemblyAfterOptimization),

include/swift/Frontend/FrontendOptions.h

@@ -137,9 +137,6 @@ class FrontendOptions {
   /// The module for which we should verify all of the generic signatures.
   std::string VerifyGenericSignaturesInModule;
 
-  /// Emit a .casid file next to the object file if CAS Backend is used.
-  bool EmitCASIDFile = false;
-
   /// CacheReplay PrefixMap.
   std::vector<std::string> CacheReplayPrefixMap;
 
@@ -389,7 +386,7 @@ class FrontendOptions {
   /// are errors. The resulting serialized AST may include errors types and
   /// skip nodes entirely, depending on the errors involved.
   bool AllowModuleWithCompilerErrors = false;
-  
+
   /// Whether or not the compiler must be strict in ensuring that implicit downstream
   /// module dependency build tasks must inherit the parent compiler invocation's context,
   /// such as `-Xcc` flags, etc.
@@ -519,7 +516,7 @@ class FrontendOptions {
 
   /// Whether we're configured to track system intermodule dependencies.
   bool shouldTrackSystemDependencies() const;
-  
+
   /// Whether we are configured with -typecheck or -typecheck-module-from-interface actuin
   bool isTypeCheckAction() const;
 
@@ -535,7 +532,7 @@ class FrontendOptions {
   ///
   /// \sa SymbolGraphASTWalker
   std::string SymbolGraphOutputDir;
-  
+
   /// Whether to emit doc comment information in symbol graphs for symbols
   /// which are inherited through classes or default implementations.
   bool SkipInheritedDocs = false;

include/swift/Option/Options.td

@@ -1657,7 +1657,7 @@ def emit_symbol_graph: Flag<["-"], "emit-symbol-graph">,
 
 def emit_symbol_graph_dir : Separate<["-"], "emit-symbol-graph-dir">,
   Flags<[FrontendOption, NoInteractiveOption, ArgumentIsPath,
-         SupplementaryOutput, HelpHidden]>,
+         SupplementaryOutput, HelpHidden, CacheInvariant]>,
   HelpText<"Emit a symbol graph to directory <dir>">,
   MetaVarName<"<dir>">;
 

lib/Frontend/CachingUtils.cpp

@@ -226,7 +226,8 @@ bool replayCachedCompilerOutputs(
                 OutputEntry{OutputPath->second, OutID, Kind, Input, *Proxy});
           return Error::success();
         })) {
-      Diag.diagnose(SourceLoc(), diag::error_cas, toString(std::move(Err)));
+      Diag.diagnose(SourceLoc(), diag::cache_replay_failed,
+                    toString(std::move(Err)));
       return lookupFailed();
     }
   };
@@ -324,8 +325,11 @@ bool replayCachedCompilerOutputs(
       }
     } else if (Output.Kind == file_types::ID::TY_Dependencies) {
       if (emitMakeDependenciesFromSerializedBuffer(
-              Output.Proxy.getData(), *File, Opts, Output.Input, Diag))
+            Output.Proxy.getData(), *File, Opts, Output.Input, Diag)) {
+        Diag.diagnose(SourceLoc(), diag::cache_replay_failed,
+                      "failed to emit dependency file");
         return false;
+      }
     } else
       *File << Output.Proxy.getData();
 

lib/Tooling/libSwiftScan/SwiftCaching.cpp

@@ -28,10 +28,12 @@
 #include "swift/Frontend/CompileJobCacheResult.h"
 #include "swift/Frontend/DiagnosticHelper.h"
 #include "swift/Frontend/Frontend.h"
+#include "swift/Frontend/MakeStyleDependencies.h"
 #include "swift/Frontend/PrintingDiagnosticConsumer.h"
 #include "swift/Option/Options.h"
 #include "clang/CAS/CASOptions.h"
 #include "clang/Frontend/CompileJobCacheResult.h"
+#include "clang/Frontend/FrontendOptions.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/SmallVector.h"
@@ -932,17 +934,20 @@ static llvm::Error replayCompilation(SwiftScanReplayInstance &Instance,
         Outputs.try_emplace(ID, File);
       });
   Outputs.try_emplace(file_types::TY_CachedDiagnostics, "<cached-diagnostics>");
+  Outputs.try_emplace(file_types::ID::TY_SymbolGraphFile, "<symbol-graph>");
 
   // Load all the output buffer.
   bool Remarks = Instance.Invocation.getCASOptions().EnableCachingRemarks;
   struct OutputEntry {
     std::string Path;
     llvm::cas::ObjectProxy Proxy;
+    file_types::ID Kind;
   };
   SmallVector<OutputEntry> OutputProxies;
   std::optional<llvm::cas::ObjectProxy> DiagnosticsOutput;
   bool UseCASBackend = Invocation.getIRGenOptions().UseCASBackend;
-  std::string ObjFile;
+  const FrontendOptions &FrontendOpts =
+      Inst.getInvocation().getFrontendOptions();
 
   swift::cas::CachedResultLoader Loader(CAS, Comp.Output);
   if (auto Err = Loader.replay(
@@ -956,15 +961,38 @@ static llvm::Error replayCompilation(SwiftScanReplayInstance &Instance,
             if (!Proxy)
               return Proxy.takeError();
 
-            if (Kind == file_types::ID::TY_Object && UseCASBackend)
-              ObjFile = OutputPath->second;
-
             if (Kind == file_types::ID::TY_CachedDiagnostics) {
               assert(!DiagnosticsOutput && "more than 1 diagnostics found");
               DiagnosticsOutput = std::move(*Proxy);
+            } else if (Kind == file_types::ID::TY_SymbolGraphFile &&
+                       !FrontendOpts.SymbolGraphOutputDir.empty()) {
+              auto Err = Proxy->forEachReference([&](llvm::cas::ObjectRef Ref)
+                                                     -> llvm::Error {
+                auto Proxy = CAS.getProxy(Ref);
+                if (!Proxy)
+                  return Proxy.takeError();
+                auto PathRef = Proxy->getReference(0);
+                auto ContentRef = Proxy->getReference(1);
+                auto Path = CAS.getProxy(PathRef);
+                auto Content = CAS.getProxy(ContentRef);
+                if (!Path)
+                  return Path.takeError();
+                if (!Content)
+                  return Content.takeError();
+
+                SmallString<128> OutputPath(FrontendOpts.SymbolGraphOutputDir);
+                llvm::sys::path::append(OutputPath, Path->getData());
+
+                OutputProxies.emplace_back(OutputEntry{
+                    std::string(OutputPath), std::move(*Content), Kind});
+
+                return Error::success();
+              });
+              if (Err)
+                return Err;
             } else
               OutputProxies.emplace_back(
-                  OutputEntry{OutputPath->second, std::move(*Proxy)});
+                  OutputEntry{OutputPath->second, std::move(*Proxy), Kind});
             return Error::success();
           }))
     return Err;
@@ -1006,11 +1034,17 @@ static llvm::Error replayCompilation(SwiftScanReplayInstance &Instance,
     auto File = Backend.createFile(Output.Path);
     if (!File)
       return File.takeError();
-    if (UseCASBackend && Output.Path == ObjFile) {
+    if (UseCASBackend && Output.Kind == file_types::ID::TY_Object) {
       auto Schema = std::make_unique<llvm::mccasformats::v1::MCSchema>(CAS);
       if (auto E = Schema->serializeObjectFile(Output.Proxy, *File))
         Inst.getDiags().diagnose(SourceLoc(), diag::error_mccas,
                                  toString(std::move(E)));
+    } else if (Output.Kind == file_types::ID::TY_Dependencies) {
+      if (emitMakeDependenciesFromSerializedBuffer(Output.Proxy.getData(),
+                                                   *File, FrontendOpts, Input,
+                                                   Inst.getDiags()))
+        Inst.getDiags().diagnose(SourceLoc(), diag::cache_replay_failed,
+                      "failed to emit dependency file");
     } else
       *File << Output.Proxy.getData();
     if (auto E = File->keep())

test/CAS/dependency_file.swift

@@ -56,6 +56,29 @@
 // RUN: %FileCheck %s --check-prefix=DEPS3 --input-file=%t/main-3.d -DTMP=%t
 // DEPS3: [[TMP]]{{/|\\}}main-3.o : [[TMP]]{{/|\\}}main.swift
 
+/// Test replay from driver interface
+// RUN: %swift-scan-test -action compute_cache_key_from_index -cas-path %t/cas -input 0 -- \
+// RUN:   %target-swift-frontend \
+// RUN:   -c -o %t/main-4.o -cache-compile-job -cas-path %t/cas -Rcache-compile-job \
+// RUN:   -swift-version 5 -disable-implicit-swift-modules \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
+// RUN:   -module-name Test -explicit-swift-module-map-file @%t/map.casid \
+// RUN:   -cache-replay-prefix-map /^src=%swift_src_root -cache-replay-prefix-map /^tmp=%t -cache-replay-prefix-map /^sdk=%t/sdk \
+// RUN:   /^tmp/main.swift @%t/MyApp-1.cmd -emit-dependencies -emit-dependencies-path %t/main-4.d > %t/key.casid
+
+// RUN: %swift-scan-test -action replay_result -cas-path %t/cas -id @%t/key.casid -- \
+// RUN:   %target-swift-frontend \
+// RUN:   -c -o %t/main-4.o -cache-compile-job -cas-path %t/cas -Rcache-compile-job \
+// RUN:   -swift-version 5 -disable-implicit-swift-modules \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
+// RUN:   -module-name Test -explicit-swift-module-map-file @%t/map.casid \
+// RUN:   -cache-replay-prefix-map /^src=%swift_src_root -cache-replay-prefix-map /^tmp=%t -cache-replay-prefix-map /^sdk=%t/sdk \
+// RUN:   /^tmp/main.swift @%t/MyApp-1.cmd -emit-dependencies -emit-dependencies-path %t/main-4.d 2>&1 | %FileCheck %s --check-prefix=CACHE-HIT4
+// CACHE-HIT4: remark: replay output file '{{.*}}{{/|\\}}main-4.d': key 'llvmcas://{{.*}}'
+// RUN: %FileCheck %s --check-prefix=DEPS4 --input-file=%t/main-4.d -DTMP=%t
+// DEPS4: [[TMP]]{{/|\\}}main-4.o : [[TMP]]{{/|\\}}main.swift
+
+
 //--- main.swift
 import A
 

test/CAS/symbol-graph.swift

@@ -23,13 +23,30 @@
 // RUN:   -parse-stdlib -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
 // RUN:   %t/main.swift -o %t/Test.swiftmodule -swift-version 5 -cache-compile-job -cas-path %t/cas -I %t/include \
 // RUN:   -emit-symbol-graph -emit-symbol-graph-dir %t/symbol-graph2 \
-// RUN:   -emit-module @%t/Test.cmd -Rcache-compile-job 2>&1 | %FileCheck %s --check-prefix=CACHE-MISS
+// RUN:   -emit-module @%t/Test.cmd -Rcache-compile-job 2>&1 | %FileCheck %s --check-prefix=CACHE-HIT
 
 // CACHE-HIT: remark: replay output file '{{.*}}{{/|\\}}symbol-graph2{{/|\\}}Test.symbols.json': key 'llvmcas://{{.*}}'
 // CACHE-HIT: remark: replay output file '{{.*}}{{/|\\}}symbol-graph2{{/|\\}}[email protected]': key 'llvmcas://{{.*}}'
 
 // RUN: diff -r -u %t/symbol-graph1 %t/symbol-graph2
 
+/// Test replay from driver interface
+// RUN: %swift-scan-test -action compute_cache_key_from_index -cas-path %t/cas -input 0 -- \
+// RUN:   %target-swift-frontend -module-name Test -module-cache-path %t/clang-module-cache -O \
+// RUN:   -parse-stdlib -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %t/main.swift -o %t/Test.swiftmodule -swift-version 5 -cache-compile-job -cas-path %t/cas -I %t/include \
+// RUN:   -emit-symbol-graph -emit-symbol-graph-dir %t/symbol-graph3 \
+// RUN:   -emit-module @%t/Test.cmd -Rcache-compile-job > %t/key.casid
+
+// RUN: %swift-scan-test -action replay_result -cas-path %t/cas -id @%t/key.casid -- \
+// RUN:   %target-swift-frontend -module-name Test -module-cache-path %t/clang-module-cache -O \
+// RUN:   -parse-stdlib -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %t/main.swift -o %t/Test.swiftmodule -swift-version 5 -cache-compile-job -cas-path %t/cas -I %t/include \
+// RUN:   -emit-symbol-graph -emit-symbol-graph-dir %t/symbol-graph3 \
+// RUN:   -emit-module @%t/Test.cmd -Rcache-compile-job
+
+// RUN: diff -r -u %t/symbol-graph1 %t/symbol-graph3
+
 //--- include/A.swiftinterface
 // swift-interface-format-version: 1.0
 // swift-module-flags: -module-name A -O -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib -user-module-version 1.0