Teach SwiftModules To Embed Incremental Information

Take advantage of the binary swiftdeps serialization utliities built during #32131. Add a new optional information block to swiftdeps files. For now, don't actually serialize swiftdeps information.

Frontends will use this information to determine whether to write incremental dependencies across modules into their swiftdeps files. We will then teach the driver to deserialize the data from this section and integrate it into its incremental decision making.

Author: CodaFi

include/swift/AST/FineGrainedDependencyFormat.h

@@ -124,7 +124,11 @@ bool readFineGrainedDependencyGraph(llvm::StringRef path,
 
 /// Tries to write the dependency graph to the given path name.
 /// Returns true if there was an error.
-bool writeFineGrainedDependencyGraph(DiagnosticEngine &diags, llvm::StringRef path,
+bool writeFineGrainedDependencyGraphToPath(DiagnosticEngine &diags,
+                                           llvm::StringRef path,
+                                           const SourceFileDepGraph &g);
+
+void writeFineGrainedDependencyGraph(llvm::BitstreamWriter &Out,
                                      const SourceFileDepGraph &g);
 
 } // namespace fine_grained_dependencies

include/swift/Subsystems.h

@@ -75,6 +75,10 @@ namespace swift {
     class TypeConverter;
   }
 
+  namespace fine_grained_dependencies {
+    class SourceFileDepGraph;
+  }
+
   /// @{
 
   /// \returns true if the declaration should be verified.  This can return
@@ -174,8 +178,10 @@ namespace swift {
   using ModuleOrSourceFile = PointerUnion<ModuleDecl *, SourceFile *>;
 
   /// Serializes a module or single source file to the given output file.
-  void serialize(ModuleOrSourceFile DC, const SerializationOptions &options,
-                 const SILModule *M = nullptr);
+  void
+  serialize(ModuleOrSourceFile DC, const SerializationOptions &options,
+            const SILModule *M = nullptr,
+            const fine_grained_dependencies::SourceFileDepGraph *DG = nullptr);
 
   /// Serializes a module or single source file to the given output file and
   /// returns back the file's contents as a memory buffer.

lib/AST/FineGrainedDependencyFormat.cpp

@@ -289,16 +289,13 @@ class Serializer {
   unsigned LastIdentifierID = 0;
   std::vector<StringRef> IdentifiersToWrite;
 
-  SmallVector<char, 0> Buffer;
-  llvm::BitstreamWriter Out{Buffer};
+  llvm::BitstreamWriter &Out;
 
   /// A reusable buffer for emitting records.
   SmallVector<uint64_t, 64> ScratchRecord;
 
   std::array<unsigned, 256> AbbrCodes;
 
-  void writeFineGrainedDependencyGraph(const SourceFileDepGraph &g);
-
   void addIdentifier(StringRef str);
   unsigned getIdentifier(StringRef str);
 
@@ -321,8 +318,10 @@ class Serializer {
   void writeMetadata();
 
 public:
-  void writeFineGrainedDependencyGraph(llvm::raw_ostream &os,
-                                       const SourceFileDepGraph &g);
+  Serializer(llvm::BitstreamWriter &ExistingOut) : Out(ExistingOut) {}
+
+public:
+  void writeFineGrainedDependencyGraph(const SourceFileDepGraph &g);
 };
 
 } // end namespace
@@ -467,20 +466,22 @@ unsigned Serializer::getIdentifier(StringRef str) {
   return iter->second;
 }
 
-void Serializer::writeFineGrainedDependencyGraph(llvm::raw_ostream &os,
-                                                 const SourceFileDepGraph &g) {
-  writeFineGrainedDependencyGraph(g);
-  os.write(Buffer.data(), Buffer.size());
-  os.flush();
+void swift::fine_grained_dependencies::writeFineGrainedDependencyGraph(
+    llvm::BitstreamWriter &Out, const SourceFileDepGraph &g) {
+  Serializer serializer{Out};
+  serializer.writeFineGrainedDependencyGraph(g);
 }
 
-bool swift::fine_grained_dependencies::writeFineGrainedDependencyGraph(
+bool swift::fine_grained_dependencies::writeFineGrainedDependencyGraphToPath(
     DiagnosticEngine &diags, StringRef path,
     const SourceFileDepGraph &g) {
   PrettyStackTraceStringAction stackTrace("saving fine-grained dependency graph", path);
   return withOutputFile(diags, path, [&](llvm::raw_ostream &out) {
-    Serializer serializer;
-    serializer.writeFineGrainedDependencyGraph(out, g);
+    SmallVector<char, 0> Buffer;
+    llvm::BitstreamWriter Writer{Buffer};
+    writeFineGrainedDependencyGraph(Writer, g);
+    out.write(Buffer.data(), Buffer.size());
+    out.flush();
     return false;
   });
-}
+}

lib/AST/FrontendSourceFileDepGraphFactory.cpp

@@ -251,7 +251,7 @@ bool fine_grained_dependencies::emitReferenceDependencies(
                              SF, outputPath, depTracker, alsoEmitDotFile)
                               .construct();
 
-  bool hadError = writeFineGrainedDependencyGraph(diags, outputPath, g);
+  bool hadError = writeFineGrainedDependencyGraphToPath(diags, outputPath, g);
 
   // If path is stdout, cannot read it back, so check for "-"
   assert(outputPath == "-" || g.verifyReadsWhatIsWritten(outputPath));

lib/Serialization/CMakeLists.txt

@@ -8,6 +8,7 @@ add_swift_host_library(swiftSerialization STATIC
   SerializedModuleLoader.cpp
   SerializedSILLoader.cpp
   SerializeDoc.cpp
+  SerializeIncremental.cpp
   SerializeSIL.cpp
 
   LLVM_LINK_COMPONENTS

lib/Serialization/ModuleFileSharedCore.cpp

@@ -1427,6 +1427,17 @@ ModuleFileSharedCore::ModuleFileSharedCore(
       break;
     }
 
+    case INCREMENTAL_INFORMATION_BLOCK_ID: {
+      HasIncrementalInfo = true;
+      // Skip incremental info if present. The Frontend currently doesn't do
+      // anything with this.
+      if (cursor.SkipBlock()) {
+        info.status = error(Status::Malformed);
+        return;
+      }
+      break;
+    }
+
     default:
       // Unknown top-level block, possibly for use by a future version of the
       // module format.

lib/Serialization/ModuleFormat.h

@@ -55,7 +55,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t SWIFTMODULE_VERSION_MINOR = 578; // Remove hasNonPatternBindingInit
+const uint16_t SWIFTMODULE_VERSION_MINOR = 579; // incremental deps info block
 
 /// A standard hash seed used for all string hashes in a serialized module.
 ///
@@ -729,6 +729,11 @@ enum BlockID {
   ///
   /// \sa decl_locs_block
   DECL_LOCS_BLOCK_ID,
+
+  /// The incremental dependency information block.
+  ///
+  /// This is part of a stable format and should not be renumbered.
+  INCREMENTAL_INFORMATION_BLOCK_ID = 196,
 };
 
 /// The record types within the control block.

lib/Serialization/Serialization.cpp

@@ -5212,9 +5212,11 @@ SerializerBase::SerializerBase(ArrayRef<unsigned char> signature,
   this->SF = DC.dyn_cast<SourceFile *>();
 }
 
-void Serializer::writeToStream(raw_ostream &os, ModuleOrSourceFile DC,
-                               const SILModule *SILMod,
-                               const SerializationOptions &options) {
+void Serializer::writeToStream(
+    raw_ostream &os, ModuleOrSourceFile DC,
+    const SILModule *SILMod,
+    const SerializationOptions &options,
+    const fine_grained_dependencies::SourceFileDepGraph *DepGraph) {
   Serializer S{SWIFTMODULE_SIGNATURE, DC};
 
   // FIXME: This is only really needed for debugging. We don't actually use it.
@@ -5226,6 +5228,9 @@ void Serializer::writeToStream(raw_ostream &os, ModuleOrSourceFile DC,
     S.writeInputBlock(options);
     S.writeSIL(SILMod, options.SerializeAllSIL);
     S.writeAST(DC);
+    if (options.ExperimentalCrossModuleIncrementalInfo) {
+      S.writeIncrementalInfo(DepGraph);
+    }
   }
 
   S.writeToStream(os);
@@ -5244,7 +5249,8 @@ void swift::serializeToBuffers(
                                "Serialization, swiftmodule, to buffer");
     llvm::SmallString<1024> buf;
     llvm::raw_svector_ostream stream(buf);
-    Serializer::writeToStream(stream, DC, M, options);
+    Serializer::writeToStream(stream, DC, M, options,
+                              /*dependency info*/ nullptr);
     bool hadError = withOutputFile(getContext(DC).Diags,
                                    options.OutputPath,
                                    [&](raw_ostream &out) {
@@ -5295,12 +5301,13 @@ void swift::serializeToBuffers(
 
 void swift::serialize(ModuleOrSourceFile DC,
                       const SerializationOptions &options,
-                      const SILModule *M) {
+                      const SILModule *M,
+                      const fine_grained_dependencies::SourceFileDepGraph *DG) {
   assert(!StringRef::withNullAsEmpty(options.OutputPath).empty());
 
   if (StringRef(options.OutputPath) == "-") {
     // Special-case writing to stdout.
-    Serializer::writeToStream(llvm::outs(), DC, M, options);
+    Serializer::writeToStream(llvm::outs(), DC, M, options, DG);
     assert(StringRef::withNullAsEmpty(options.DocOutputPath).empty());
     return;
   }
@@ -5310,7 +5317,7 @@ void swift::serialize(ModuleOrSourceFile DC,
                                  [&](raw_ostream &out) {
     FrontendStatsTracer tracer(getContext(DC).Stats,
                                "Serialization, swiftmodule");
-    Serializer::writeToStream(out, DC, M, options);
+    Serializer::writeToStream(out, DC, M, options, DG);
     return false;
   });
   if (hadError)

lib/Serialization/Serialization.h

@@ -34,6 +34,10 @@ namespace clang {
 namespace swift {
   class SILModule;
 
+  namespace fine_grained_dependencies {
+    class SourceFileDepGraph;
+  }
+
 namespace serialization {
 
 using FilenamesTy = ArrayRef<std::string>;
@@ -389,14 +393,21 @@ class Serializer : public SerializerBase {
   /// Top-level entry point for serializing a module.
   void writeAST(ModuleOrSourceFile DC);
 
+  /// Serializes the given dependnecy graph into the incremental information
+  /// section of this swift module.
+  void writeIncrementalInfo(
+      const fine_grained_dependencies::SourceFileDepGraph *DepGraph);
+
   using SerializerBase::SerializerBase;
   using SerializerBase::writeToStream;
 
 public:
   /// Serialize a module to the given stream.
-  static void writeToStream(raw_ostream &os, ModuleOrSourceFile DC,
-                            const SILModule *M,
-                            const SerializationOptions &options);
+  static void
+  writeToStream(raw_ostream &os, ModuleOrSourceFile DC,
+                const SILModule *M,
+                const SerializationOptions &options,
+                const fine_grained_dependencies::SourceFileDepGraph *DepGraph);
 
   /// Records the use of the given Type.
   ///

lib/Serialization/SerializeIncremental.cpp

@@ -0,0 +1,34 @@
+//===--- SerializeIncremental.cpp - Write incremental swiftdeps -----------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "serialize-incremental"
+#include "Serialization.h"
+#include "swift/AST/FineGrainedDependencyFormat.h"
+
+#include <type_traits>
+
+using namespace swift;
+using namespace swift::serialization;
+using namespace llvm::support;
+using llvm::BCBlockRAII;
+
+void Serializer::writeIncrementalInfo(
+    const fine_grained_dependencies::SourceFileDepGraph *DepGraph) {
+  if (!DepGraph)
+    return;
+
+  {
+    BCBlockRAII restoreBlock(Out, INCREMENTAL_INFORMATION_BLOCK_ID, 5);
+    swift::fine_grained_dependencies::writeFineGrainedDependencyGraph(
+        Out, *DepGraph);
+  }
+}