Merge pull request swiftlang#19935 from graydon/text-interface-to-module

swift-ci · web-flow · commit 7a7657b0f329 · 2018-10-20T22:22:47.000-07:00
diff --git a/include/swift/AST/FileSystem.h b/include/swift/AST/FileSystem.h
@@ -0,0 +1,44 @@
+//===--- FileSystem.h - File helpers that interact with Diags ---*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_AST_FILESYSTEM_H
+#define SWIFT_AST_FILESYSTEM_H
+
+#include "swift/Basic/FileSystem.h"
+#include "swift/AST/DiagnosticEngine.h"
+
+namespace swift {
+
+/// A wrapper around swift::atomicallyWritingToFile that handles diagnosing any
+/// filesystem errors and asserts the output path is nonempty.
+///
+/// \returns true if there were any errors, either from the filesystem
+/// operations or from \p action returning true.
+inline bool
+withOutputFile(DiagnosticEngine &diags, StringRef outputPath,
+               llvm::function_ref<bool(llvm::raw_pwrite_stream &)> action) {
+  assert(!outputPath.empty());
+
+  bool actionFailed = false;
+  std::error_code EC = swift::atomicallyWritingToFile(
+      outputPath,
+      [&](llvm::raw_pwrite_stream &out) { actionFailed = action(out); });
+  if (EC) {
+    diags.diagnose(SourceLoc(), diag::error_opening_output, outputPath,
+                   EC.message());
+    return true;
+  }
+  return actionFailed;
+}
+} // end namespace swift
+
+#endif // SWIFT_AST_FILESYSTEM_H
diff --git a/include/swift/Basic/FileTypes.def b/include/swift/Basic/FileTypes.def
@@ -50,6 +50,8 @@ TYPE("swiftmodule",         SwiftModuleFile,           "swiftmodule",     "")
 TYPE("swiftdoc",            SwiftModuleDocFile,        "swiftdoc",        "")
 TYPE("swiftinterface",      SwiftParseableInterfaceFile, \
                                                        "swiftinterface",  "")
+TYPE("swiftinterfacedeps",  SwiftParseableInterfaceDeps, \
+                                                       "sid",             "")
 TYPE("assembly",            Assembly,                  "s",               "")
 TYPE("raw-sil",             RawSIL,                    "sil",             "")
 TYPE("raw-sib",             RawSIB,                    "sib",             "")
diff --git a/include/swift/Frontend/ParseableInterfaceSupport.h b/include/swift/Frontend/ParseableInterfaceSupport.h
@@ -64,9 +64,10 @@ class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase {
   std::string CacheDir;
 
   void
-  configureSubInvocationAndOutputPath(CompilerInvocation &SubInvocation,
-                                      StringRef InPath,
-                                      llvm::SmallString<128> &OutPath);
+  configureSubInvocationAndOutputPaths(CompilerInvocation &SubInvocation,
+                                       StringRef InPath,
+                                       llvm::SmallString<128> &OutPath,
+                                       llvm::SmallString<128> &DepPath);
 
   std::error_code
   openModuleFiles(StringRef DirName, StringRef ModuleFilename,
diff --git a/lib/Basic/FileTypes.cpp b/lib/Basic/FileTypes.cpp
@@ -79,6 +79,7 @@ bool file_types::isTextual(ID Id) {
   case file_types::TY_ModuleTrace:
   case file_types::TY_OptRecord:
   case file_types::TY_SwiftParseableInterfaceFile:
+  case file_types::TY_SwiftParseableInterfaceDeps:
     return true;
   case file_types::TY_Image:
   case file_types::TY_Object:
@@ -135,6 +136,7 @@ bool file_types::isAfterLLVM(ID Id) {
   case file_types::TY_ModuleTrace:
   case file_types::TY_OptRecord:
   case file_types::TY_SwiftParseableInterfaceFile:
+  case file_types::TY_SwiftParseableInterfaceDeps:
     return false;
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID.");
@@ -167,6 +169,7 @@ bool file_types::isPartOfSwiftCompilation(ID Id) {
   case file_types::TY_SwiftModuleFile:
   case file_types::TY_SwiftModuleDocFile:
   case file_types::TY_SwiftParseableInterfaceFile:
+  case file_types::TY_SwiftParseableInterfaceDeps:
   case file_types::TY_SerializedDiagnostics:
   case file_types::TY_ClangModuleFile:
   case file_types::TY_SwiftDeps:
diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp
@@ -1780,6 +1780,7 @@ void Driver::buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
       case file_types::TY_ModuleTrace:
       case file_types::TY_OptRecord:
       case file_types::TY_SwiftParseableInterfaceFile:
+      case file_types::TY_SwiftParseableInterfaceDeps:
         // We could in theory handle assembly or LLVM input, but let's not.
         // FIXME: What about LTO?
         Diags.diagnose(SourceLoc(), diag::error_unexpected_input_file,
diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp
@@ -438,6 +438,7 @@ const char *ToolChain::JobContext::computeFrontendModeForCompile() const {
   case file_types::TY_TBD:
   case file_types::TY_OptRecord:
   case file_types::TY_SwiftParseableInterfaceFile:
+  case file_types::TY_SwiftParseableInterfaceDeps:
     llvm_unreachable("Output type can never be primary output.");
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID");
@@ -670,6 +671,7 @@ ToolChain::constructInvocation(const BackendJobAction &job,
     case file_types::TY_ModuleTrace:
     case file_types::TY_OptRecord:
     case file_types::TY_SwiftParseableInterfaceFile:
+    case file_types::TY_SwiftParseableInterfaceDeps:
       llvm_unreachable("Output type can never be primary output.");
     case file_types::TY_INVALID:
       llvm_unreachable("Invalid type ID");
diff --git a/lib/Frontend/ParseableInterfaceSupport.cpp b/lib/Frontend/ParseableInterfaceSupport.cpp
@@ -14,12 +14,14 @@
 #include "swift/AST/ASTContext.h"
 #include "swift/AST/Decl.h"
 #include "swift/AST/DiagnosticsFrontend.h"
+#include "swift/AST/FileSystem.h"
 #include "swift/AST/Module.h"
 #include "swift/Frontend/Frontend.h"
 #include "swift/Frontend/ParseableInterfaceSupport.h"
 #include "swift/SILOptimizer/PassManager/Passes.h"
 #include "swift/Serialization/SerializationOptions.h"
 #include "clang/Basic/Module.h"
+#include "llvm/ADT/Hashing.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/CrashRecoveryContext.h"
@@ -31,6 +33,7 @@ using namespace swift;
 
 #define SWIFT_TOOLS_VERSION_KEY "swift-tools-version"
 #define SWIFT_MODULE_FLAGS_KEY "swift-module-flags"
+#define SWIFT_INTERFACE_DEPS_VERSION "swift-interface-deps-version-1"
 
 static bool
 extractSwiftInterfaceVersionAndArgs(DiagnosticEngine &Diags,
@@ -66,11 +69,44 @@ extractSwiftInterfaceVersionAndArgs(DiagnosticEngine &Diags,
   return false;
 }
 
+/// Construct a cache key for the .swiftmodule being generated. There is a
+/// balance to be struck here between things that go in the cache key and
+/// things that go in the "up to date" check of the cache entry. We want to
+/// avoid fighting over a single cache entry too much when (say) running
+/// different compiler versions on the same machine or different inputs
+/// that happen to have the same short module name, so we will disambiguate
+/// those in the key. But we want to invalidate and rebuild a cache entry
+/// -- rather than making a new one and potentially filling up the cache
+/// with dead entries -- when other factors change, such as the contents of
+/// the .swiftinterface input or its dependencies.
+std::string getCacheHash(ASTContext &Ctx,
+                         CompilerInvocation &SubInvocation,
+                         StringRef InPath) {
+  // Start with the compiler version (which will be either tag names or revs).
+  std::string vers = swift::version::getSwiftFullVersion(
+      Ctx.LangOpts.EffectiveLanguageVersion);
+  llvm::hash_code H = llvm::hash_value(vers);
+
+  // Simplest representation of input "identity" (not content) is just a
+  // pathname, and probably all we can get from the VFS in this regard anyways.
+  H = llvm::hash_combine(H, InPath);
+
+  // ClangImporterOpts does include the target CPU, which is redundant: we
+  // already have separate .swiftinterface files per target due to expanding
+  // preprocessing directives, but further specializing the cache key to that
+  // target is harmless and will not make any extra cache entries, so allow it.
+  H = llvm::hash_combine(
+      H, SubInvocation.getClangImporterOptions().getPCHHashComponents());
+
+  return llvm::APInt(64, H).toString(36, /*Signed=*/false);
+}
+
 void
-ParseableInterfaceModuleLoader::configureSubInvocationAndOutputPath(
+ParseableInterfaceModuleLoader::configureSubInvocationAndOutputPaths(
     CompilerInvocation &SubInvocation,
     StringRef InPath,
-    llvm::SmallString<128> &OutPath) {
+    llvm::SmallString<128> &OutPath,
+    llvm::SmallString<128> &DepPath) {
 
   auto &SearchPathOpts = Ctx.SearchPathOpts;
   auto &LangOpts = Ctx.LangOpts;
@@ -84,16 +120,19 @@ ParseableInterfaceModuleLoader::configureSubInvocationAndOutputPath(
   SubInvocation.setRuntimeResourcePath(SearchPathOpts.RuntimeResourcePath);
   SubInvocation.setTargetTriple(LangOpts.Target);
 
-  // Calculate an output filename based on the SubInvocation hash, and
-  // wire up the SubInvocation's InputsAndOutputs to contain both
-  // input and output filenames.
+  // Calculate an output filename that includes a hash of relevant key data, and
+  // wire up the SubInvocation's InputsAndOutputs to contain both input and
+  // output filenames.
   OutPath = CacheDir;
   llvm::sys::path::append(OutPath, llvm::sys::path::stem(InPath));
   OutPath.append("-");
-  OutPath.append(SubInvocation.getPCHHash());
+  OutPath.append(getCacheHash(Ctx, SubInvocation, InPath));
   OutPath.append(".");
-  auto Ext = file_types::getExtension(file_types::TY_SwiftModuleFile);
-  OutPath.append(Ext);
+  DepPath = OutPath;
+  auto OutExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
+  OutPath.append(OutExt);
+  auto DepExt = file_types::getExtension(file_types::TY_SwiftParseableInterfaceDeps);
+  DepPath.append(DepExt);
 
   auto &FEOpts = SubInvocation.getFrontendOptions();
   FEOpts.RequestedAction = FrontendOptions::ActionType::EmitModuleOnly;
@@ -104,24 +143,67 @@ ParseableInterfaceModuleLoader::configureSubInvocationAndOutputPath(
   FEOpts.InputsAndOutputs.setMainAndSupplementaryOutputs({MainOut}, {SOPs});
 }
 
-// FIXME: this needs to be a more extensive up-to-date check.
+// Write the world's simplest dependencies file: a version identifier on
+// a line followed by a list of files, one per line.
+static bool writeSwiftInterfaceDeps(DiagnosticEngine &Diags,
+                                    ArrayRef<std::string> Deps,
+                                    StringRef DepPath) {
+  return withOutputFile(Diags, DepPath, [&](llvm::raw_pwrite_stream &out) {
+      out << SWIFT_INTERFACE_DEPS_VERSION << '\n';
+      for (auto const &D : Deps) {
+        out << D << '\n';
+      }
+      return false;
+    });
+}
+
+// Check that the output .swiftmodule file is at least as new as all the
+// dependencies it read when it was built last time.
 static bool
 swiftModuleIsUpToDate(clang::vfs::FileSystem &FS,
-                      StringRef InPath, StringRef OutPath) {
-  if (FS.exists(OutPath)) {
-    auto InStatus = FS.status(InPath);
-    auto OutStatus = FS.status(OutPath);
-    if (InStatus && OutStatus) {
-      return InStatus.get().getLastModificationTime() <=
-        OutStatus.get().getLastModificationTime();
+                      StringRef InPath, StringRef OutPath, StringRef DepPath) {
+
+  if (!FS.exists(OutPath) || !FS.exists(DepPath))
+    return false;
+
+  auto OutStatus = FS.status(OutPath);
+  if (!OutStatus)
+    return false;
+
+  auto DepBuf = FS.getBufferForFile(DepPath);
+  if (!DepBuf)
+    return false;
+
+  // Split the deps file into a vector of lines.
+  StringRef Deps =  DepBuf.get()->getBuffer();
+  SmallVector<StringRef, 16> AllDeps;
+  Deps.split(AllDeps, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
+
+  // First line in vector is a version-string; check it is the expected value.
+  if (AllDeps.size() < 1 ||
+      AllDeps[0] != SWIFT_INTERFACE_DEPS_VERSION) {
+    return false;
+  }
+
+  // Overwrite the version-string entry in the vector with the .swiftinterface
+  // input file we're reading, then stat() every entry in the vector and check
+  // none are newer than the .swiftmodule (OutStatus).
+  AllDeps[0] = InPath;
+  for (auto In : AllDeps) {
+    auto InStatus = FS.status(In);
+    if (!InStatus ||
+        (InStatus.get().getLastModificationTime() >
+         OutStatus.get().getLastModificationTime())) {
+      return false;
     }
   }
-  return false;
+  return true;
 }
 
 static bool buildSwiftModuleFromSwiftInterface(
     clang::vfs::FileSystem &FS, DiagnosticEngine &Diags,
-    CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath) {
+    CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath,
+    StringRef DepPath) {
   bool SubError = false;
   bool RunSuccess = llvm::CrashRecoveryContext().RunSafelyOnThread([&] {
 
@@ -145,6 +227,7 @@ static bool buildSwiftModuleFromSwiftInterface(
     // module-serialization task we're trying to do here.
     LLVM_DEBUG(llvm::dbgs() << "Setting up instance\n");
     CompilerInstance SubInstance;
+    SubInstance.createDependencyTracker(/*TrackSystemDeps=*/false);
     if (SubInstance.setup(SubInvocation)) {
       SubError = true;
       return;
@@ -179,6 +262,12 @@ static bool buildSwiftModuleFromSwiftInterface(
       });
     SILMod->serialize();
     SubError = Diags.hadAnyError();
+
+    if (!SubError) {
+      SubError |= writeSwiftInterfaceDeps(
+          Diags, SubInstance.getDependencyTracker()->getDependencies(),
+          DepPath);
+    }
   });
   return !RunSuccess || SubError;
 }
@@ -194,7 +283,7 @@ std::error_code ParseableInterfaceModuleLoader::openModuleFiles(
 
   auto &FS = *Ctx.SourceMgr.getFileSystem();
   auto &Diags = Ctx.Diags;
-  llvm::SmallString<128> InPath, OutPath;
+  llvm::SmallString<128> InPath, OutPath, DepPath;
 
   // First check to see if the .swiftinterface exists at all. Bail if not.
   InPath = DirName;
@@ -207,12 +296,12 @@ std::error_code ParseableInterfaceModuleLoader::openModuleFiles(
   // Set up a _potential_ sub-invocation to consume the .swiftinterface and emit
   // the .swiftmodule.
   CompilerInvocation SubInvocation;
-  configureSubInvocationAndOutputPath(SubInvocation, InPath, OutPath);
+  configureSubInvocationAndOutputPaths(SubInvocation, InPath, OutPath, DepPath);
 
   // Evaluate if we need to run this sub-invocation, and if so run it.
-  if (!swiftModuleIsUpToDate(FS, InPath, OutPath)) {
+  if (!swiftModuleIsUpToDate(FS, InPath, OutPath, DepPath)) {
     if (buildSwiftModuleFromSwiftInterface(FS, Diags, SubInvocation, InPath,
-                                           OutPath))
+                                           OutPath, DepPath))
       return std::make_error_code(std::errc::invalid_argument);
   }
 
diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp
@@ -29,6 +29,7 @@
 #include "swift/AST/ASTScope.h"
 #include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/AST/DiagnosticsSema.h"
+#include "swift/AST/FileSystem.h"
 #include "swift/AST/GenericSignatureBuilder.h"
 #include "swift/AST/IRGenOptions.h"
 #include "swift/AST/ASTMangler.h"
@@ -312,30 +313,6 @@ static bool writeSIL(SILModule &SM, const PrimarySpecificPaths &PSPs,
                   PSPs.OutputFilename, opts.EmitSortedSIL);
 }
 
-/// A wrapper around swift::atomicallyWritingToFile that handles diagnosing any
-/// filesystem errors and ignores empty output paths.
-///
-/// \returns true if there were any errors, either from the filesystem
-/// operations or from \p action returning true.
-static bool atomicallyWritingToTextFile(
-    StringRef outputPath, DiagnosticEngine &diags,
-    llvm::function_ref<bool(llvm::raw_pwrite_stream &)> action) {
-  assert(!outputPath.empty());
-
-  bool actionFailed = false;
-  std::error_code EC =
-      swift::atomicallyWritingToFile(outputPath,
-                                     [&](llvm::raw_pwrite_stream &out) {
-    actionFailed = action(out);
-  });
-  if (EC) {
-    diags.diagnose(SourceLoc(), diag::error_opening_output,
-                   outputPath, EC.message());
-    return true;
-  }
-  return actionFailed;
-}
-
 /// Prints the Objective-C "generated header" interface for \p M to \p
 /// outputPath.
 ///
@@ -348,8 +325,8 @@ static bool printAsObjCIfNeeded(StringRef outputPath, ModuleDecl *M,
                                 StringRef bridgingHeader, bool moduleIsPublic) {
   if (outputPath.empty())
     return false;
-  return atomicallyWritingToTextFile(outputPath, M->getDiags(),
-                                     [&](raw_ostream &out) -> bool {
+  return withOutputFile(M->getDiags(), outputPath,
+                        [&](raw_ostream &out) -> bool {
     auto requiredAccess = moduleIsPublic ? AccessLevel::Public
                                          : AccessLevel::Internal;
     return printAsObjC(out, M, bridgingHeader, requiredAccess);
@@ -368,8 +345,8 @@ static bool printParseableInterfaceIfNeeded(StringRef outputPath,
                                             ModuleDecl *M) {
   if (outputPath.empty())
     return false;
-  return atomicallyWritingToTextFile(outputPath, M->getDiags(),
-                                     [M, Opts](raw_ostream &out) -> bool {
+  return withOutputFile(M->getDiags(), outputPath,
+                        [M, Opts](raw_ostream &out) -> bool {
     return swift::emitParseableInterface(out, Opts, M);
   });
 }
diff --git a/lib/FrontendTool/ReferenceDependencies.cpp b/lib/FrontendTool/ReferenceDependencies.cpp
diff --git a/lib/Serialization/Serialization.cpp b/lib/Serialization/Serialization.cpp
diff --git a/test/ParseableInterface/client.swift b/test/ParseableInterface/client.swift
