[ModuleInterface] Switch from mtime to xxhash in FILE_DEPENDENCY records.

Author: graydon

include/swift/Serialization/ModuleFormat.h

@@ -52,7 +52,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t SWIFTMODULE_VERSION_MINOR = 466; // Last change: add isAutoClosure flag to param
+const uint16_t SWIFTMODULE_VERSION_MINOR = 467; // Last change: switch FILE_DEPENDENCY records to hashes.
 
 using DeclIDField = BCFixed<31>;
 
@@ -108,6 +108,7 @@ using CharOffsetField = BitOffsetField;
 
 using FileSizeField = BCVBR<16>;
 using FileModTimeField = BCVBR<16>;
+using FileHashField = BCVBR<16>;
 
 // These IDs must \em not be renumbered or reordered without incrementing
 // the module version.
@@ -652,7 +653,7 @@ namespace input_block {
     IMPORTED_HEADER,
     BCFixed<1>, // exported?
     FileSizeField, // file size (for validation)
-    FileModTimeField, // file mtime (for validation)
+    FileHashField, // file hash (for validation)
     BCBlob // file path
   >;
 

include/swift/Serialization/SerializationOptions.h

@@ -37,7 +37,7 @@ namespace swift {
 
     struct FileDependency {
       uint64_t Size;
-      llvm::sys::TimePoint<> LastModTime;
+      uint64_t Hash;
       StringRef Path;
     };
     ArrayRef<FileDependency> Dependencies;

lib/Frontend/ParseableInterfaceSupport.cpp

@@ -27,6 +27,7 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "llvm/ADT/Hashing.h"
+#include "llvm/Support/xxhash.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/CrashRecoveryContext.h"
@@ -73,6 +74,22 @@ extractSwiftInterfaceVersionAndArgs(DiagnosticEngine &Diags,
   return false;
 }
 
+static bool
+getHashOfFile(clang::vfs::FileSystem &FS,
+              StringRef Path, uint64_t &HashOut,
+              DiagnosticEngine &Diags) {
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buf =
+    FS.getBufferForFile(Path, /*FileSize=*/-1,
+                        /*RequiresNullTerminator=*/false);
+  if (!Buf) {
+    Diags.diagnose(SourceLoc(), diag::cannot_open_file, Path,
+                   Buf.getError().message());
+    return true;
+  }
+  HashOut = xxHash64(Buf.get()->getBuffer());
+  return false;
+}
+
 /// Construct a cache key for the .swiftmodule being generated. There is a
 /// balance to be struck here between things that go in the cache key and
 /// things that go in the "up to date" check of the cache entry. We want to
@@ -83,9 +100,9 @@ extractSwiftInterfaceVersionAndArgs(DiagnosticEngine &Diags,
 /// -- rather than making a new one and potentially filling up the cache
 /// with dead entries -- when other factors change, such as the contents of
 /// the .swiftinterface input or its dependencies.
-std::string getCacheHash(ASTContext &Ctx,
-                         CompilerInvocation &SubInvocation,
-                         StringRef InPath) {
+static std::string getCacheHash(ASTContext &Ctx,
+                                CompilerInvocation &SubInvocation,
+                                StringRef InPath) {
   // Start with the compiler version (which will be either tag names or revs).
   std::string vers = swift::version::getSwiftFullVersion(
       Ctx.LangOpts.EffectiveLanguageVersion);
@@ -154,7 +171,8 @@ ParseableInterfaceModuleLoader::configureSubInvocationAndOutputPaths(
 static bool
 swiftModuleIsUpToDate(clang::vfs::FileSystem &FS,
                       StringRef ModuleCachePath,
-                      StringRef OutPath) {
+                      StringRef OutPath,
+                      DiagnosticEngine &Diags) {
 
   if (!FS.exists(OutPath))
     return false;
@@ -177,10 +195,12 @@ swiftModuleIsUpToDate(clang::vfs::FileSystem &FS,
     return false;
 
   for (auto In : AllDeps) {
+    uint64_t Hash = 0;
     auto InStatus = FS.status(In.Path);
     if (!InStatus ||
         (InStatus.get().getSize() != In.Size) ||
-        (InStatus.get().getLastModificationTime() != In.LastModTime)) {
+        getHashOfFile(FS, In.Path, Hash, Diags) ||
+        (Hash != In.Hash)) {
       LLVM_DEBUG(llvm::dbgs() << "Dep " << In.Path
                  << " is directly out of date\n");
       return false;
@@ -190,7 +210,7 @@ swiftModuleIsUpToDate(clang::vfs::FileSystem &FS,
     auto Ty = file_types::lookupTypeForExtension(Ext);
     if (Ty == file_types::TY_SwiftModuleFile &&
         In.Path.startswith(ModuleCachePath) &&
-        !swiftModuleIsUpToDate(FS, ModuleCachePath, In.Path)) {
+        !swiftModuleIsUpToDate(FS, ModuleCachePath, In.Path, Diags)) {
       LLVM_DEBUG(llvm::dbgs() << "Dep " << In.Path
                  << " is indirectly out of date\n");
       return false;
@@ -276,9 +296,13 @@ static bool buildSwiftModuleFromSwiftInterface(
         SubError = true;
         return;
       }
+      uint64_t Hash = 0;
+      if (getHashOfFile(FS, Dep, Hash, Diags)) {
+        SubError = true;
+        return;
+      }
       Deps.push_back(SerializationOptions::FileDependency{
-          DepStatus.get().getSize(), DepStatus.get().getLastModificationTime(),
-          Dep});
+          DepStatus.get().getSize(), Hash, Dep});
     }
     serializationOpts.Dependencies = Deps;
     SILMod->setSerializeSILAction([&]() {
@@ -317,7 +341,7 @@ std::error_code ParseableInterfaceModuleLoader::openModuleFiles(
   configureSubInvocationAndOutputPaths(SubInvocation, InPath, OutPath);
 
   // Evaluate if we need to run this sub-invocation, and if so run it.
-  if (!swiftModuleIsUpToDate(FS, CacheDir, OutPath)) {
+  if (!swiftModuleIsUpToDate(FS, CacheDir, OutPath, Diags)) {
     if (buildSwiftModuleFromSwiftInterface(FS, Diags, SubInvocation, InPath,
                                            OutPath))
       return std::make_error_code(std::errc::invalid_argument);

lib/Serialization/ModuleFile.cpp

@@ -254,7 +254,7 @@ static bool validateInputBlock(
     switch (kind) {
     case input_block::FILE_DEPENDENCY:
       dependencies.push_back(SerializationOptions::FileDependency{
-          scratch[0], llvm::sys::toTimePoint(scratch[1]), blobData});
+          scratch[0], scratch[1], blobData});
       break;
     default:
       // Unknown metadata record, possibly for use by a future version of the

lib/Serialization/Serialization.cpp

@@ -1053,9 +1053,7 @@ void Serializer::writeInputBlock(const SerializationOptions &options) {
   }
 
   for (auto const &dep : options.Dependencies) {
-    FileDependency.emit(ScratchRecord, dep.Size,
-                        llvm::sys::toTimeT(dep.LastModTime),
-                        dep.Path);
+    FileDependency.emit(ScratchRecord, dep.Size, dep.Hash, dep.Path);
   }
 
   SmallVector<ModuleDecl::ImportedModule, 8> allImports;

test/ParseableInterface/ModuleCache/module-cache-intermediate-mtime-change-rebuilds-only-intermediate.swift renamed to test/ParseableInterface/ModuleCache/module-cache-intermediate-hash-change-rebuilds-only-intermediate.swift

@@ -4,7 +4,7 @@
 // Setup builds a module TestModule that depends on OtherModule and LeafModule (built from other.swift and leaf.swift).
 // During setup, input and intermediate mtimes are all set to a constant 'old' time (long in the past).
 //
-// We then modify OtherModule.swiftinterface, and check only OtherModule-*.swiftmodule has a new mtime, LeafModule is unchanged.
+// We then modify OtherModule.swiftinterface's content (but not size), and check only OtherModule-*.swiftmodule has a new mtime, LeafModule is unchanged.
 //
 //
 // Setup phase 1: Write input files.
@@ -13,6 +13,7 @@
 //
 // RUN: echo 'import LeafModule' >%t/other.swift
 // RUN: echo 'public func OtherFunc() -> Int { return LeafFunc(); }' >>%t/other.swift
+// RUN: echo 'public func OtherFunc2() -> Int { return LeafFunc(); }' >>%t/other.swift
 //
 //
 // Setup phase 2: build modules, pushing timestamps of inputs and intermediates into the past as we go.
@@ -26,12 +27,12 @@
 // RUN: %S/Inputs/make-old.py %t/modulecache/OtherModule-*.swiftmodule
 //
 //
-// Actual test: make OtherModule.swiftinterface newer, check we only rebuild its cached module.
+// Actual test: Change a byte in OtherModule.swiftinterface, check we only rebuild its cached module.
 //
 // RUN: %S/Inputs/check-is-old.py %t/OtherModule.swiftinterface %t/LeafModule.swiftinterface
 // RUN: %S/Inputs/check-is-old.py %t/modulecache/OtherModule-*.swiftmodule %t/modulecache/LeafModule-*.swiftmodule
-// RUN: touch %t/OtherModule.swiftinterface
-// RUN: %S/Inputs/check-is-new.py %t/OtherModule.swiftinterface
+// RUN: sed -e 's/OtherFunc2/OtterFunc2/' -i.prev %t/OtherModule.swiftinterface
+// RUN: %S/Inputs/make-old.py %t/OtherModule.swiftinterface
 // RUN: rm %t/TestModule.swiftmodule
 // RUN: %target-swift-frontend -I %t -module-cache-path %t/modulecache -enable-parseable-module-interface -emit-module -o %t/TestModule.swiftmodule -module-name TestModule %s
 // RUN: %S/Inputs/check-is-new.py %t/modulecache/OtherModule-*.swiftmodule

test/ParseableInterface/ModuleCache/module-cache-leaf-mtime-change-rebuilds-all.swift renamed to test/ParseableInterface/ModuleCache/module-cache-leaf-hash-change-rebuilds-all.swift

@@ -4,12 +4,13 @@
 // Setup builds a module TestModule that depends on OtherModule and LeafModule (built from other.swift and leaf.swift).
 // During setup, input and intermediate mtimes are all set to a constant 'old' time (long in the past).
 //
-// We then modify LeafModule.swiftinterface, and check both cached modules have new mtimes.
+// We then modify LeafModule.swiftinterface's content (but not size), and check both cached modules have new mtimes.
 //
 //
 // Setup phase 1: Write input files.
 //
 // RUN: echo 'public func LeafFunc() -> Int { return 10; }' >%t/leaf.swift
+// RUN: echo 'public func LeafFunc2() -> Int { return 11; }' >>%t/leaf.swift
 //
 // RUN: echo 'import LeafModule' >%t/other.swift
 // RUN: echo 'public func OtherFunc() -> Int { return LeafFunc(); }' >>%t/other.swift
@@ -26,12 +27,12 @@
 // RUN: %S/Inputs/make-old.py %t/modulecache/OtherModule-*.swiftmodule
 //
 //
-// Actual test: make LeafModule.swiftinterface newer, check both cached modules get rebuilt.
+// Actual test: Change a byte in LeafModule.swiftinterface, check both cached modules get rebuilt.
 //
 // RUN: %S/Inputs/check-is-old.py %t/OtherModule.swiftinterface %t/LeafModule.swiftinterface
 // RUN: %S/Inputs/check-is-old.py %t/modulecache/OtherModule-*.swiftmodule %t/modulecache/LeafModule-*.swiftmodule
-// RUN: touch %t/LeafModule.swiftinterface
-// RUN: %S/Inputs/check-is-new.py %t/LeafModule.swiftinterface
+// RUN: sed -e 's/LeafFunc2/LoafFunc2/' -i.prev %t/LeafModule.swiftinterface
+// RUN: %S/Inputs/make-old.py %t/LeafModule.swiftinterface
 // RUN: rm %t/TestModule.swiftmodule
 // RUN: %target-swift-frontend -I %t -module-cache-path %t/modulecache -enable-parseable-module-interface -emit-module -o %t/TestModule.swiftmodule -module-name TestModule %s
 // RUN: %S/Inputs/check-is-new.py %t/modulecache/OtherModule-*.swiftmodule %t/modulecache/LeafModule-*.swiftmodule