[Serialization] Allow either hash-based or mtime-based deps

Add a bit to the module to determine whether the dependency’s stored bit pattern is a hash or an mtime.

Prebuilt modules store a hash of their dependencies because we can’t be sure their dependencies will have the same modtime as when they were built.

Author: Harlan Haskins

include/swift/Serialization/ModuleFormat.h

@@ -52,7 +52,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t SWIFTMODULE_VERSION_MINOR = 475; // Last change: Generalize nested archetype serialization
+const uint16_t SWIFTMODULE_VERSION_MINOR = 476; // Last change: prebuilt module cache
 
 using DeclIDField = BCFixed<31>;
 
@@ -107,7 +107,7 @@ using CharOffset = BitOffset;
 using CharOffsetField = BitOffsetField;
 
 using FileSizeField = BCVBR<16>;
-using FileModTimeField = BCVBR<16>;
+using FileModTimeOrContentHashField = BCVBR<16>;
 using FileHashField = BCVBR<16>;
 
 // These IDs must \em not be renumbered or reordered without incrementing
@@ -671,9 +671,10 @@ namespace input_block {
 
   using FileDependencyLayout = BCRecordLayout<
     FILE_DEPENDENCY,
-    FileSizeField,    // file size (for validation)
-    FileModTimeField, // file mtime (for validation)
-    BCBlob            // path
+    FileSizeField,                 // file size (for validation)
+    FileModTimeOrContentHashField, // mtime or content hash (for validation)
+    BCFixed<1>,                    // are we reading mtime (0) or hash (1)?
+    BCBlob                         // path
   >;
 }
 

include/swift/Serialization/SerializationOptions.h

@@ -35,10 +35,87 @@ namespace swift {
     StringRef ModuleLinkName;
     ArrayRef<std::string> ExtraClangOptions;
 
-    struct FileDependency {
-      uint64_t Size;
-      uint64_t ModificationTime;
+    /// Describes a single-file dependency for this module, along with the
+    /// appropriate strategy for how to verify if it's up-to-date.
+    class FileDependency {
+      /// The size of the file on disk, in bytes.
+      uint64_t Size : 63;
+
+      /// A dependency can be either hash-based or modification-time-based.
+      bool IsHashBased : 1;
+
+      union {
+        /// The last modification time of the file.
+        uint64_t ModificationTime;
+
+        /// The xxHash of the full contents of the file.
+        uint64_t ContentHash;
+      };
+
+      /// The path to the dependency.
       std::string Path;
+
+      FileDependency(uint64_t size, bool isHash, uint64_t hashOrModTime,
+                     StringRef path):
+        Size(size), IsHashBased(isHash), ModificationTime(hashOrModTime),
+        Path(path) {}
+    public:
+      FileDependency() = delete;
+
+      /// Creates a new hash-based file dependency.
+      static FileDependency
+      hashBased(StringRef path, uint64_t size, uint64_t hash) {
+        return FileDependency(size, /*isHash*/true, hash, path);
+      }
+
+      /// Creates a new modification time-based file dependency.
+      static FileDependency
+      modTimeBased(StringRef path, uint64_t size, uint64_t mtime) {
+        return FileDependency(size, /*isHash*/false, mtime, path);
+      }
+
+      /// Updates the last-modified time of this dependency.
+      /// If the dependency is a hash-based dependency, it becomes
+      /// modification time-based.
+      void setLastModificationTime(uint64_t mtime) {
+        IsHashBased = false;
+        ModificationTime = mtime;
+      }
+
+      /// Updates the content hash of this dependency.
+      /// If the dependency is a modification time-based dependency, it becomes
+      /// hash-based.
+      void setContentHash(uint64_t hash) {
+        IsHashBased = true;
+        ContentHash = hash;
+      }
+
+      /// Determines if this dependency is hash-based and should be validated
+      /// based on content hash.
+      bool isHashBased() const { return IsHashBased; }
+
+      /// Determines if this dependency is hash-based and should be validated
+      /// based on modification time.
+      bool isModificationTimeBased() const { return !IsHashBased; }
+
+      /// Gets the modification time, if this is a modification time-based
+      /// dependency.
+      uint64_t getModificationTime() const {
+        assert(isModificationTimeBased() &&
+               "cannot get modification time for hash-based dependency");
+        return ModificationTime;
+      }
+
+      /// Gets the content hash, if this is a hash-based
+      /// dependency.
+      uint64_t getContentHash() const {
+        assert(isHashBased() &&
+               "cannot get content hash for mtime-based dependency");
+        return ContentHash;
+      }
+
+      StringRef getPath() const { return Path; }
+      uint64_t getSize() const { return Size; }
     };
     ArrayRef<FileDependency> Dependencies;
 

lib/Frontend/ParseableInterfaceSupport.cpp

@@ -225,14 +225,40 @@ void ParseableInterfaceModuleLoader::configureSubInvocationInputsAndOutputs(
 }
 
 // Checks that a dependency read from the cached module is up to date compared
-// to the interface file it represents.
-static bool dependencyIsUpToDate(llvm::vfs::FileSystem &FS, FileDependency In,
-                                 StringRef ModulePath, DiagnosticEngine &Diags,
-                                 SourceLoc DiagLoc) {
-  auto Status = getStatusOfDependency(FS, ModulePath, In.Path, Diags, DiagLoc);
-  if (!Status) return false;
-  uint64_t mtime = Status->getLastModificationTime().time_since_epoch().count();
-  return Status->getSize() == In.Size && mtime == In.ModificationTime;
+// to the interface file it represents. If it's up-to-date, return its status,
+// so we can switch hash-based dependencies to mtimes after we've validated
+// the hashes.
+static Optional<llvm::vfs::Status>
+dependencyIsUpToDate(llvm::vfs::FileSystem &FS, FileDependency In,
+                     StringRef ModulePath, DiagnosticEngine &Diags,
+                     SourceLoc DiagLoc) {
+  auto Status = getStatusOfDependency(FS, ModulePath, In.getPath(),
+                                      Diags, DiagLoc);
+  if (!Status) return None;
+
+  // If the sizes differ, then we know the file has changed.
+  if (Status->getSize() != In.getSize()) return None;
+
+  // Otherwise, if this dependency is verified by modification time, check
+  // it vs. the modification time of the file.
+  uint64_t mtime =
+    Status->getLastModificationTime().time_since_epoch().count();
+
+  if (In.isModificationTimeBased())
+    return mtime == In.getModificationTime() ? Status : None;
+
+  // Slow path: if the dependency is verified by content hash, check it vs. the
+  // hash of the file.
+  auto buf = getBufferOfDependency(FS, ModulePath, In.getPath(),
+                                   Diags, DiagLoc);
+  if (!buf) return None;
+
+  if (xxHash64(buf->getBuffer()) == In.getContentHash())
+    return Status;
+
+  return None;
+}
+
 }
 
 // Check that the output .swiftmodule file is at least as new as all the
@@ -289,6 +315,7 @@ collectDepsForSerialization(llvm::vfs::FileSystem &FS,
                             CompilerInstance &SubInstance,
                             StringRef InPath, StringRef ModuleCachePath,
                             SmallVectorImpl<FileDependency> &Deps,
+                            bool IsHashBased,
                             DiagnosticEngine &Diags, SourceLoc DiagLoc,
                             DependencyTracker *OuterTracker) {
   auto DTDeps = SubInstance.getDependencyTracker()->getDependencies();
@@ -297,17 +324,25 @@ collectDepsForSerialization(llvm::vfs::FileSystem &FS,
   llvm::StringSet<> AllDepNames;
   for (auto const &DepName : InitialDepNames) {
     if (AllDepNames.insert(DepName).second && OuterTracker) {
-        OuterTracker->addDependency(DepName, /*IsSystem=*/false);
+      OuterTracker->addDependency(DepName, /*IsSystem=*/false);
     }
-    auto Status = getStatusOfDependency(FS, InPath, DepName, Diags, DiagLoc);
-    if (!Status)
-      return true;
     auto DepBuf = getBufferOfDependency(FS, InPath, DepName, Diags, DiagLoc);
     if (!DepBuf)
       return true;
-    uint64_t mtime =
-      Status->getLastModificationTime().time_since_epoch().count();
-    Deps.push_back(FileDependency{Status->getSize(), mtime, DepName});
+    auto Status = getStatusOfDependency(FS, InPath, DepName, Diags, DiagLoc);
+    if (!Status)
+      return true;
+
+    if (IsHashBased) {
+      uint64_t hash = xxHash64(DepBuf->getBuffer());
+      Deps.push_back(
+        FileDependency::hashBased(DepName, Status->getSize(), hash));
+    } else {
+      uint64_t mtime =
+        Status->getLastModificationTime().time_since_epoch().count();
+      Deps.push_back(
+        FileDependency::modTimeBased(DepName, Status->getSize(), mtime));
+    }
 
     if (ModuleCachePath.empty())
       continue;
@@ -331,10 +366,10 @@ collectDepsForSerialization(llvm::vfs::FileSystem &FS,
         return true;
       }
       for (auto const &SubDep : SubDeps) {
-        if (AllDepNames.insert(SubDep.Path).second) {
+        if (AllDepNames.insert(SubDep.getPath()).second) {
           Deps.push_back(SubDep);
           if (OuterTracker)
-            OuterTracker->addDependency(SubDep.Path, /*IsSystem=*/false);
+            OuterTracker->addDependency(SubDep.getPath(), /*IsSystem=*/false);
         }
       }
     }

lib/Serialization/ModuleFile.cpp

@@ -252,10 +252,19 @@ static bool validateInputBlock(
     StringRef blobData;
     unsigned kind = cursor.readRecord(entry.ID, scratch, &blobData);
     switch (kind) {
-    case input_block::FILE_DEPENDENCY:
-      dependencies.push_back(SerializationOptions::FileDependency{
-          scratch[0], scratch[1], blobData});
+    case input_block::FILE_DEPENDENCY: {
+      bool isHashBased = scratch[2] != 0;
+      if (isHashBased) {
+        dependencies.push_back(
+          SerializationOptions::FileDependency::hashBased(
+            blobData, scratch[0], scratch[1]));
+      } else {
+        dependencies.push_back(
+          SerializationOptions::FileDependency::modTimeBased(
+            blobData, scratch[0], scratch[1]));
+      }
       break;
+    }
     default:
       // Unknown metadata record, possibly for use by a future version of the
       // module format.

lib/Serialization/Serialization.cpp

@@ -1037,6 +1037,11 @@ static void flattenImportPath(const ModuleDecl::ImportedModule &import,
   out.append(accessPathElem.first.str());
 }
 
+uint64_t getRawModTimeOrHash(const SerializationOptions::FileDependency &dep) {
+  if (dep.isHashBased()) return dep.getContentHash();
+  return dep.getModificationTime();
+}
+
 void Serializer::writeInputBlock(const SerializationOptions &options) {
   BCBlockRAII restoreBlock(Out, INPUT_BLOCK_ID, 4);
   input_block::ImportedModuleLayout ImportedModule(Out);
@@ -1058,9 +1063,11 @@ void Serializer::writeInputBlock(const SerializationOptions &options) {
   }
 
   for (auto const &dep : options.Dependencies) {
-    FileDependency.emit(ScratchRecord, dep.Size,
-                        dep.ModificationTime,
-                        dep.Path);
+    FileDependency.emit(ScratchRecord,
+                        dep.getSize(),
+                        getRawModTimeOrHash(dep),
+                        dep.isHashBased(),
+                        dep.getPath());
   }
 
   SmallVector<ModuleDecl::ImportedModule, 8> allImports;