Merge pull request #35394 from rintaro/serialization-source-file-list

rintaro · web-flow · commit 9a93a3b40f64 · 2021-01-15T10:01:46.000-08:00
[Serialization] Serialize source file information to '.swiftsourceinfo' file
diff --git a/include/swift/AST/FileUnit.h b/include/swift/AST/FileUnit.h
@@ -402,6 +402,9 @@ class LoadedFile : public FileUnit {
     return false;
   }
 
+  virtual void collectBasicSourceFileInfo(
+      llvm::function_ref<void(const BasicSourceFileInfo &)> callback) const {}
+
   static bool classof(const FileUnit *file) {
     return file->getKind() == FileUnitKind::SerializedAST ||
            file->getKind() == FileUnitKind::ClangModule ||
diff --git a/include/swift/AST/Module.h b/include/swift/AST/Module.h
@@ -728,6 +728,10 @@ class ModuleDecl : public DeclContext, public TypeDecl {
     return ReverseFullNameIterator(this);
   }
 
+  /// Calls \p callback for each source file of the module.
+  void collectBasicSourceFileInfo(
+      llvm::function_ref<void(const BasicSourceFileInfo &)> callback);
+
   SourceRange getSourceRange() const { return SourceRange(); }
 
   static bool classof(const DeclContext *DC) {
diff --git a/include/swift/AST/RawComment.h b/include/swift/AST/RawComment.h
@@ -13,11 +13,15 @@
 #ifndef SWIFT_AST_RAW_COMMENT_H
 #define SWIFT_AST_RAW_COMMENT_H
 
+#include "swift/Basic/Fingerprint.h"
 #include "swift/Basic/LLVM.h"
 #include "swift/Basic/SourceLoc.h"
 #include "swift/Basic/SourceManager.h"
 
 namespace swift {
+
+class SourceFile;
+
 struct SingleRawComment {
   enum class CommentKind {
     OrdinaryLine,  ///< Any normal // comments
@@ -92,6 +96,17 @@ struct BasicDeclLocs {
   LineColumn EndLoc;
 };
 
+struct BasicSourceFileInfo {
+  StringRef FilePath;
+  Fingerprint InterfaceHash = Fingerprint::ZERO();
+  llvm::sys::TimePoint<> LastModified = {};
+  uint64_t FileSize = 0;
+
+  BasicSourceFileInfo() {}
+
+  bool populate(SourceFile *SF);
+};
+
 } // namespace swift
 
 #endif // LLVM_SWIFT_AST_RAW_COMMENT_H
diff --git a/include/swift/AST/SourceFile.h b/include/swift/AST/SourceFile.h
@@ -531,6 +531,9 @@ class SourceFile final : public FileUnit {
     out << getInterfaceHash() << '\n';
   }
 
+  /// Get this file's interface hash including the type members in the file.
+  Fingerprint getInterfaceHashIncludingTypeMembers() const;
+
   /// If this source file has been told to collect its parsed tokens, retrieve
   /// those tokens.
   ArrayRef<Token> getAllTokens() const;
diff --git a/include/swift/Serialization/SerializedModuleLoader.h b/include/swift/Serialization/SerializedModuleLoader.h
@@ -446,6 +446,9 @@ class SerializedASTFile final : public LoadedFile {
 
   StringRef getTargetTriple() const;
 
+  virtual void collectBasicSourceFileInfo(
+      llvm::function_ref<void(const BasicSourceFileInfo &)>) const override;
+
   static bool classof(const FileUnit *file) {
     return file->getKind() == FileUnitKind::SerializedAST;
   }
diff --git a/lib/AST/Module.cpp b/lib/AST/Module.cpp
@@ -1099,6 +1099,30 @@ Fingerprint SourceFile::getInterfaceHash() const {
   return Fingerprint{std::move(result)};
 }
 
+Fingerprint SourceFile::getInterfaceHashIncludingTypeMembers() const {
+  /// FIXME: Gross. Hashing multiple "hash" values.
+  llvm::MD5 hash;
+  hash.update(getInterfaceHash().getRawValue());
+
+  std::function<void(IterableDeclContext *)> hashTypeBodyFingerprints =
+      [&](IterableDeclContext *IDC) {
+        if (auto fp = IDC->getBodyFingerprint())
+          hash.update(fp->getRawValue());
+        for (auto *member : IDC->getParsedMembers())
+          if (auto *childIDC = dyn_cast<IterableDeclContext>(member))
+            hashTypeBodyFingerprints(childIDC);
+      };
+
+  for (auto *D : getTopLevelDecls()) {
+    if (auto IDC = dyn_cast<IterableDeclContext>(D))
+      hashTypeBodyFingerprints(IDC);
+  }
+
+  llvm::MD5::MD5Result result;
+  hash.final(result);
+  return Fingerprint{std::move(result)};
+}
+
 syntax::SourceFileSyntax SourceFile::getSyntaxRoot() const {
   assert(shouldBuildSyntaxTree() && "Syntax tree disabled");
   auto &eval = getASTContext().evaluator;
@@ -1514,6 +1538,20 @@ const clang::Module *ModuleDecl::findUnderlyingClangModule() const {
   return nullptr;
 }
 
+void ModuleDecl::collectBasicSourceFileInfo(
+    llvm::function_ref<void(const BasicSourceFileInfo &)> callback) {
+  for (FileUnit *fileUnit : getFiles()) {
+    if (SourceFile *SF = dyn_cast<SourceFile>(fileUnit)) {
+      BasicSourceFileInfo info;
+      if (info.populate(SF))
+        continue;
+      callback(info);
+    } else if (auto *serialized = dyn_cast<LoadedFile>(fileUnit)) {
+      serialized->collectBasicSourceFileInfo(callback);
+    }
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // Cross-Import Overlays
 //===----------------------------------------------------------------------===//
diff --git a/lib/AST/RawComment.cpp b/lib/AST/RawComment.cpp
@@ -22,6 +22,7 @@
 #include "swift/AST/FileUnit.h"
 #include "swift/AST/Module.h"
 #include "swift/AST/PrettyStackTrace.h"
+#include "swift/AST/SourceFile.h"
 #include "swift/AST/Types.h"
 #include "swift/Basic/PrimitiveParsing.h"
 #include "swift/Basic/SourceManager.h"
@@ -238,3 +239,27 @@ CharSourceRange RawComment::getCharSourceRange() {
                 static_cast<const char *>(Start.getOpaquePointerValue());
   return CharSourceRange(Start, Length);
 }
+
+bool BasicSourceFileInfo::populate(SourceFile *SF) {
+  SourceManager &SM = SF->getASTContext().SourceMgr;
+
+  auto filename = SF->getFilename();
+  if (filename.empty())
+    return true;
+  auto stat = SM.getFileSystem()->status(filename);
+  if (!stat)
+    return true;
+
+  FilePath = filename;
+  LastModified = stat->getLastModificationTime();
+  FileSize = stat->getSize();
+
+  if (SF->hasInterfaceHash()) {
+    InterfaceHash = SF->getInterfaceHashIncludingTypeMembers();
+  } else {
+    // FIXME: Parse the file with EnableInterfaceHash option.
+    InterfaceHash = Fingerprint::ZERO();
+  }
+
+  return false;
+}
diff --git a/lib/IDE/CompletionInstance.cpp b/lib/IDE/CompletionInstance.cpp
@@ -277,49 +277,6 @@ static bool areAnyDependentFilesInvalidated(
       });
 }
 
-/// Get interface hash of \p SF including the type members in the file.
-///
-/// See if the inteface of the function and types visible from a function body
-/// has changed since the last completion. If they haven't changed, completion
-/// can reuse the existing AST of the source file. \c SF->getInterfaceHash() is
-/// not enough because it doesn't take the interface of the type members into
-/// account. For example:
-///
-///   struct S {
-///     func foo() {}
-///   }
-///   func main(val: S) {
-///     val.<HERE>
-///   }
-///
-/// In this case, we need to ensure that the interface of \c S hasn't changed.
-/// Note that we don't care about local types (i.e. type declarations inside
-/// function bodies, closures, or top level statement bodies) because they are
-/// not visible from other functions where the completion is happening.
-static Fingerprint getInterfaceHashIncludingTypeMembers(const SourceFile *SF) {
-  /// FIXME: Gross. Hashing multiple "hash" values.
-  llvm::MD5 hash;
-  hash.update(SF->getInterfaceHash().getRawValue());
-
-  std::function<void(IterableDeclContext *)> hashTypeBodyFingerprints =
-      [&](IterableDeclContext *IDC) {
-        if (auto fp = IDC->getBodyFingerprint())
-          hash.update(fp->getRawValue());
-        for (auto *member : IDC->getParsedMembers())
-          if (auto *childIDC = dyn_cast<IterableDeclContext>(member))
-            hashTypeBodyFingerprints(childIDC);
-      };
-
-  for (auto *D : SF->getTopLevelDecls()) {
-    if (auto IDC = dyn_cast<IterableDeclContext>(D))
-      hashTypeBodyFingerprints(IDC);
-  }
-
-  llvm::MD5::MD5Result result;
-  hash.final(result);
-  return Fingerprint{std::move(result)};
-}
-
 } // namespace
 
 bool CompletionInstance::performCachedOperationIfPossible(
@@ -396,8 +353,26 @@ bool CompletionInstance::performCachedOperationIfPossible(
   switch (newInfo.Kind) {
   case CodeCompletionDelayedDeclKind::FunctionBody: {
     // If the interface has changed, AST must be refreshed.
-    const auto oldInterfaceHash = getInterfaceHashIncludingTypeMembers(oldSF);
-    const auto newInterfaceHash = getInterfaceHashIncludingTypeMembers(tmpSF);
+    // See if the inteface of the function and types visible from a function
+    // body has changed since the last completion. If they haven't changed,
+    // completion can reuse the existing AST of the source file.
+    // \c getInterfaceHash() is not enough because it doesn't take the interface
+    // of the type members into account. For example:
+    //
+    //   struct S {
+    //     func foo() {}
+    //   }
+    //   func main(val: S) {
+    //     val.<HERE>
+    //   }
+    //
+    // In this case, we need to ensure that the interface of \c S hasn't
+    // changed. Note that we don't care about local types (i.e. type
+    // declarations inside function bodies, closures, or top level statement
+    // bodies) because they are not visible from other functions where the
+    // completion is happening.
+    const auto oldInterfaceHash = oldSF->getInterfaceHashIncludingTypeMembers();
+    const auto newInterfaceHash = tmpSF->getInterfaceHashIncludingTypeMembers();
     if (oldInterfaceHash != newInterfaceHash)
       return false;
 
diff --git a/lib/Serialization/ModuleFile.cpp b/lib/Serialization/ModuleFile.cpp
@@ -959,6 +959,41 @@ Optional<CommentInfo> ModuleFile::getCommentForDecl(const Decl *D) const {
   return getCommentForDeclByUSR(USRBuffer.str());
 }
 
+void ModuleFile::collectBasicSourceFileInfo(
+    llvm::function_ref<void(const BasicSourceFileInfo &)> callback) const {
+  if (Core->SourceFileListData.empty())
+    return;
+  assert(!Core->SourceLocsTextData.empty());
+
+  auto *Cursor = Core->SourceFileListData.bytes_begin();
+  auto *End = Core->SourceFileListData.bytes_end();
+  while (Cursor < End) {
+    // FilePath (byte offset in 'SourceLocsTextData').
+    auto fileID = endian::readNext<uint32_t, little, unaligned>(Cursor);
+    // InterfaceHash (fixed length string).
+    auto fpStr = StringRef{reinterpret_cast<const char *>(Cursor),
+                           Fingerprint::DIGEST_LENGTH};
+    Cursor += Fingerprint::DIGEST_LENGTH;
+    // LastModified (nanoseconds since epoch).
+    auto timestamp = endian::readNext<uint64_t, little, unaligned>(Cursor);
+    // FileSize (num of bytes).
+    auto fileSize = endian::readNext<uint64_t, little, unaligned>(Cursor);
+
+    assert(fileID < Core->SourceLocsTextData.size());
+    auto filePath = Core->SourceLocsTextData.substr(fileID);
+    size_t terminatorOffset = filePath.find('\0');
+    filePath = filePath.slice(0, terminatorOffset);
+
+    BasicSourceFileInfo info;
+    info.FilePath = filePath;
+    info.InterfaceHash = Fingerprint::fromString(fpStr);
+    info.LastModified =
+        llvm::sys::TimePoint<>(std::chrono::nanoseconds(timestamp));
+    info.FileSize = fileSize;
+    callback(info);
+  }
+}
+
 Optional<BasicDeclLocs>
 ModuleFile::getBasicDeclLocsForDecl(const Decl *D) const {
   assert(D);
diff --git a/lib/Serialization/ModuleFile.h b/lib/Serialization/ModuleFile.h
@@ -694,6 +694,8 @@ class ModuleFile
   Optional<BasicDeclLocs> getBasicDeclLocsForDecl(const Decl *D) const;
   Identifier getDiscriminatorForPrivateValue(const ValueDecl *D);
   Optional<Fingerprint> loadFingerprint(const IterableDeclContext *IDC) const;
+  void collectBasicSourceFileInfo(
+      llvm::function_ref<void(const BasicSourceFileInfo &)> callback) const;
 
 
   // MARK: Deserialization interface
diff --git a/lib/Serialization/ModuleFileSharedCore.cpp b/lib/Serialization/ModuleFileSharedCore.cpp
@@ -974,6 +974,9 @@ bool ModuleFileSharedCore::readDeclLocsBlock(llvm::BitstreamCursor &cursor) {
         return false;
       }
       switch (*kind) {
+      case decl_locs_block::SOURCE_FILE_LIST:
+        SourceFileListData = blobData;
+        break;
       case decl_locs_block::BASIC_DECL_LOCS:
         BasicDeclLocsData = blobData;
         break;
diff --git a/lib/Serialization/ModuleFileSharedCore.h b/lib/Serialization/ModuleFileSharedCore.h
@@ -289,6 +289,9 @@ class ModuleFileSharedCore {
   /// A blob of 0 terminated string segments referenced in \c SourceLocsTextData
   StringRef SourceLocsTextData;
 
+  /// A blob of source file list.
+  StringRef SourceFileListData;
+
   /// An array of fixed size source location data for each USR appearing in
   /// \c DeclUSRsTable.
   StringRef BasicDeclLocsData;
diff --git a/lib/Serialization/SerializeDoc.cpp b/lib/Serialization/SerializeDoc.cpp
@@ -783,6 +783,66 @@ static void emitBasicLocsRecord(llvm::BitstreamWriter &Out,
   DeclLocsList.emit(scratch, Writer.Buffer);
 }
 
+static void emitFileListRecord(llvm::BitstreamWriter &Out,
+                               ModuleOrSourceFile MSF, StringWriter &FWriter) {
+  assert(MSF);
+
+  struct SourceFileListWriter {
+    StringWriter &FWriter;
+
+    llvm::SmallString<0> Buffer;
+    llvm::StringSet<> seenFilenames;
+
+    void emitSourceFileInfo(const BasicSourceFileInfo &info) {
+      // Make 'FilePath' absolute for serialization;
+      SmallString<128> absolutePath = info.FilePath;
+      llvm::sys::fs::make_absolute(absolutePath);
+
+      // Don't emit duplicated files.
+      if (!seenFilenames.insert(info.FilePath).second)
+        return;
+
+      auto fileID = FWriter.getTextOffset(absolutePath);
+      auto fingerprintStr = info.InterfaceHash.getRawValue();
+      auto timestamp = std::chrono::duration_cast<std::chrono::nanoseconds>(
+                           info.LastModified.time_since_epoch())
+                           .count();
+
+      llvm::raw_svector_ostream out(Buffer);
+      endian::Writer writer(out, little);
+      // FilePath.
+      writer.write<uint32_t>(fileID);
+      // InterfaceHash (fixed length string).
+      assert(fingerprintStr.size() == Fingerprint::DIGEST_LENGTH);
+      out << fingerprintStr;
+      // LastModified (nanoseconds since epoch).
+      writer.write<uint64_t>(timestamp);
+      // FileSize (num of bytes).
+      writer.write<uint64_t>(info.FileSize);
+    }
+
+    SourceFileListWriter(StringWriter &FWriter) : FWriter(FWriter) {
+      Buffer.reserve(1024);
+    }
+  } writer(FWriter);
+
+  if (SourceFile *SF = MSF.dyn_cast<SourceFile *>()) {
+    BasicSourceFileInfo info;
+    if (info.populate(SF))
+      return;
+    writer.emitSourceFileInfo(info);
+  } else {
+    auto *M = MSF.get<ModuleDecl *>();
+    M->collectBasicSourceFileInfo([&](const BasicSourceFileInfo &info) {
+      writer.emitSourceFileInfo(info);
+    });
+  }
+
+  const decl_locs_block::SourceFileListLayout FileList(Out);
+  SmallVector<uint64_t, 8> scratch;
+  FileList.emit(scratch, writer.Buffer);
+}
+
 class SourceInfoSerializer : public SerializerBase {
 public:
   using SerializerBase::SerializerBase;
@@ -807,6 +867,7 @@ class SourceInfoSerializer : public SerializerBase {
     BLOCK_RECORD(control_block, TARGET);
 
     BLOCK(DECL_LOCS_BLOCK);
+    BLOCK_RECORD(decl_locs_block, SOURCE_FILE_LIST);
     BLOCK_RECORD(decl_locs_block, BASIC_DECL_LOCS);
     BLOCK_RECORD(decl_locs_block, DECL_USRS);
     BLOCK_RECORD(decl_locs_block, TEXT_DATA);
@@ -849,6 +910,7 @@ void serialization::writeSourceInfoToStream(raw_ostream &os,
       DeclUSRsTableWriter USRWriter;
       StringWriter FPWriter;
       DocRangeWriter DocWriter;
+      emitFileListRecord(S.Out, DC, FPWriter);
       emitBasicLocsRecord(S.Out, DC, USRWriter, FPWriter, DocWriter);
       // Emit USR table mapping from a USR to USR Id.
       // The basic locs record uses USR Id instead of actual USR, so that we
diff --git a/lib/Serialization/SerializedModuleLoader.cpp b/lib/Serialization/SerializedModuleLoader.cpp
diff --git a/lib/Serialization/SourceInfoFormat.h b/lib/Serialization/SourceInfoFormat.h
diff --git a/test/Serialization/Inputs/SourceInfo/File1.swift b/test/Serialization/Inputs/SourceInfo/File1.swift
diff --git a/test/Serialization/Inputs/SourceInfo/File2.swift b/test/Serialization/Inputs/SourceInfo/File2.swift
diff --git a/test/Serialization/sourceinfo.swift b/test/Serialization/sourceinfo.swift
diff --git a/tools/swift-ide-test/swift-ide-test.cpp b/tools/swift-ide-test/swift-ide-test.cpp

Original file line number	Diff line number	Diff line change
`@@ -402,6 +402,9 @@ class LoadedFile : public FileUnit {`
`402`	`402`	`return false;`
`403`	`403`	`}`
`404`	`404`
	`405`	`+ virtual void collectBasicSourceFileInfo(`
	`406`	`+ llvm::function_ref<void(const BasicSourceFileInfo &)> callback) const {}`
	`407`	`+`
`405`	`408`	`static bool classof(const FileUnit *file) {`
`406`	`409`	`return file->getKind() == FileUnitKind::SerializedAST \|\|`
`407`	`410`	`file->getKind() == FileUnitKind::ClangModule \|\|`
Original file line number	Diff line number	Diff line change
`@@ -728,6 +728,10 @@ class ModuleDecl : public DeclContext, public TypeDecl {`
`728`	`728`	`return ReverseFullNameIterator(this);`
`729`	`729`	`}`
`730`	`730`
	`731`	`+ /// Calls \p callback for each source file of the module.`
	`732`	`+ void collectBasicSourceFileInfo(`
	`733`	`+ llvm::function_ref<void(const BasicSourceFileInfo &)> callback);`
	`734`	`+`
`731`	`735`	`SourceRange getSourceRange() const { return SourceRange(); }`
`732`	`736`
`733`	`737`	`static bool classof(const DeclContext *DC) {`
Original file line number	Diff line number	Diff line change
`@@ -531,6 +531,9 @@ class SourceFile final : public FileUnit {`
`531`	`531`	`out << getInterfaceHash() << '\n';`
`532`	`532`	`}`
`533`	`533`
	`534`	`+ /// Get this file's interface hash including the type members in the file.`
	`535`	`+ Fingerprint getInterfaceHashIncludingTypeMembers() const;`
	`536`	`+`
`534`	`537`	`/// If this source file has been told to collect its parsed tokens, retrieve`
`535`	`538`	`/// those tokens.`
`536`	`539`	`ArrayRef<Token> getAllTokens() const;`
Original file line number	Diff line number	Diff line change
`@@ -446,6 +446,9 @@ class SerializedASTFile final : public LoadedFile {`
`446`	`446`
`447`	`447`	`StringRef getTargetTriple() const;`
`448`	`448`
	`449`	`+ virtual void collectBasicSourceFileInfo(`
	`450`	`+ llvm::function_ref<void(const BasicSourceFileInfo &)>) const override;`
	`451`	`+`
`449`	`452`	`static bool classof(const FileUnit *file) {`
`450`	`453`	`return file->getKind() == FileUnitKind::SerializedAST;`
`451`	`454`	`}`