Frontend: set up output file .swiftsourceinfo

nkcsgexi · nkcsgexi · commit 3103b5cec16b · 2019-09-24T13:52:17.000-07:00
This patch will focus on teaching driver and frontend to emit this file. The actual content and de-serialization parts will come later. More details: https://forums.swift.org/t/proposal-emitting-source-information-file-during-compilation/28794
diff --git a/cmake/modules/SwiftSource.cmake b/cmake/modules/SwiftSource.cmake
@@ -310,9 +310,11 @@ function(_compile_swift_files
     set(module_base "${module_dir}/${SWIFTFILE_MODULE_NAME}")
     if(SWIFTFILE_SDK IN_LIST SWIFT_APPLE_PLATFORMS)
       set(specific_module_dir "${module_base}.swiftmodule")
+      set(specific_module_private_dir "${specific_module_dir}/Private")
       set(module_base "${module_base}.swiftmodule/${SWIFTFILE_ARCHITECTURE}")
     else()
       set(specific_module_dir)
+      set(specific_module_private_dir)
     endif()
     set(module_file "${module_base}.swiftmodule")
     set(module_doc_file "${module_base}.swiftdoc")
@@ -490,6 +492,7 @@ function(_compile_swift_files
         COMMAND
           "${CMAKE_COMMAND}" "-E" "make_directory" ${module_dir}
           ${specific_module_dir}
+          ${specific_module_private_dir}
         COMMAND
           "${PYTHON_EXECUTABLE}" "${line_directive_tool}" "@${file_path}" --
           "${swift_compiler_tool}" "-emit-module" "-o" "${module_file}"
diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def
@@ -124,6 +124,8 @@ ERROR(error_mode_cannot_emit_module,none,
       "this mode does not support emitting modules", ())
 ERROR(error_mode_cannot_emit_module_doc,none,
       "this mode does not support emitting module documentation files", ())
+ERROR(error_mode_cannot_emit_module_source_info,none,
+      "this mode does not support emitting module source info files", ())
 ERROR(error_mode_cannot_emit_interface,none,
       "this mode does not support emitting module interface files", ())
 
diff --git a/include/swift/Basic/FileTypes.def b/include/swift/Basic/FileTypes.def
@@ -50,6 +50,7 @@ TYPE("autolink",            AutolinkFile,              "autolink",        "")
 TYPE("swiftmodule",         SwiftModuleFile,           "swiftmodule",     "")
 TYPE("swiftdoc",            SwiftModuleDocFile,        "swiftdoc",        "")
 TYPE("swiftinterface",      SwiftModuleInterfaceFile,  "swiftinterface",  "")
+TYPE("swiftsourceinfo",     SwiftSourceInfoFile,       "swiftsourceinfo", "")
 TYPE("assembly",            Assembly,                  "s",               "")
 TYPE("raw-sil",             RawSIL,                    "sil",             "")
 TYPE("raw-sib",             RawSIB,                    "sib",             "")
diff --git a/include/swift/Basic/SupplementaryOutputPaths.h b/include/swift/Basic/SupplementaryOutputPaths.h
@@ -41,6 +41,15 @@ struct SupplementaryOutputPaths {
   /// \sa swift::serialize
   std::string ModuleOutputPath;
 
+  /// The path to which we should emit a module source information file.
+  /// It is valid whenever there are any inputs.
+  ///
+  /// This binary format stores source locations and other information about the
+  /// declarations in a module.
+  ///
+  /// \sa swift::serialize
+  std::string ModuleSourceInfoOutputPath;
+
   /// The path to which we should emit a module documentation file.
   /// It is valid whenever there are any inputs.
   ///
@@ -132,7 +141,8 @@ struct SupplementaryOutputPaths {
            ModuleDocOutputPath.empty() && DependenciesFilePath.empty() &&
            ReferenceDependenciesFilePath.empty() &&
            SerializedDiagnosticsPath.empty() && LoadedModuleTracePath.empty() &&
-           TBDPath.empty() && ModuleInterfaceOutputPath.empty();
+           TBDPath.empty() && ModuleInterfaceOutputPath.empty() &&
+           ModuleSourceInfoOutputPath.empty();
   }
 };
 } // namespace swift
diff --git a/include/swift/Driver/Driver.h b/include/swift/Driver/Driver.h
@@ -354,6 +354,12 @@ class Driver {
                                       StringRef workingDirectory,
                                       CommandOutput *Output) const;
 
+  void choosePrivateOutputFilePath(Compilation &C,
+                                   const TypeToPathMap *OutputMap,
+                                   StringRef workingDirectory,
+                                   CommandOutput *Output,
+                                   file_types::ID fileID) const;
+
   void chooseModuleInterfacePath(Compilation &C, const JobAction *JA,
                                  StringRef workingDirectory,
                                  llvm::SmallString<128> &buffer,
diff --git a/include/swift/Frontend/FrontendInputsAndOutputs.h b/include/swift/Frontend/FrontendInputsAndOutputs.h
@@ -235,6 +235,7 @@ class FrontendInputsAndOutputs {
   bool hasLoadedModuleTracePath() const;
   bool hasModuleOutputPath() const;
   bool hasModuleDocOutputPath() const;
+  bool hasModuleSourceInfoOutputPath() const;
   bool hasModuleInterfaceOutputPath() const;
   bool hasTBDPath() const;
 
diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td
@@ -38,6 +38,11 @@ def emit_module_doc_path
   : Separate<["-"], "emit-module-doc-path">, MetaVarName<"<path>">,
     HelpText<"Output module documentation file <path>">;
 
+def emit_module_source_info : Flag<["-"], "emit-module-source-info">,
+  HelpText<"Output module source info file">;
+def emit_module_source_info_path : Separate<["-"], "emit-module-source-info-path">,
+  HelpText<"Output module source info file to <path>">;
+
 def merge_modules
   : Flag<["-"], "merge-modules">, ModeOpt,
     HelpText<"Merge the input modules without otherwise processing them">;
diff --git a/include/swift/Serialization/SerializationOptions.h b/include/swift/Serialization/SerializationOptions.h
@@ -29,6 +29,7 @@ namespace swift {
 
     const char *OutputPath = nullptr;
     const char *DocOutputPath = nullptr;
+    const char *SourceInfoOutputPath = nullptr;
 
     StringRef GroupInfoPath;
     StringRef ImportedHeader;
diff --git a/include/swift/Subsystems.h b/include/swift/Subsystems.h
@@ -282,6 +282,7 @@ namespace swift {
                           const SerializationOptions &opts,
                           std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
                           std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
+                          std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
                           const SILModule *M = nullptr);
 
   /// Get the CPU, subtarget feature options, and triple to use when emitting code.
diff --git a/lib/Basic/FileTypes.cpp b/lib/Basic/FileTypes.cpp
@@ -89,6 +89,7 @@ bool file_types::isTextual(ID Id) {
   case file_types::TY_RawSIB:
   case file_types::TY_SwiftModuleFile:
   case file_types::TY_SwiftModuleDocFile:
+  case file_types::TY_SwiftSourceInfoFile:
   case file_types::TY_LLVM_BC:
   case file_types::TY_SerializedDiagnostics:
   case file_types::TY_ClangModuleFile:
@@ -128,6 +129,7 @@ bool file_types::isAfterLLVM(ID Id) {
   case file_types::TY_RawSIB:
   case file_types::TY_SwiftModuleFile:
   case file_types::TY_SwiftModuleDocFile:
+  case file_types::TY_SwiftSourceInfoFile:
   case file_types::TY_SerializedDiagnostics:
   case file_types::TY_ClangModuleFile:
   case file_types::TY_SwiftDeps:
@@ -169,6 +171,7 @@ bool file_types::isPartOfSwiftCompilation(ID Id) {
   case file_types::TY_SwiftModuleFile:
   case file_types::TY_SwiftModuleDocFile:
   case file_types::TY_SwiftModuleInterfaceFile:
+  case file_types::TY_SwiftSourceInfoFile:
   case file_types::TY_SerializedDiagnostics:
   case file_types::TY_ClangModuleFile:
   case file_types::TY_SwiftDeps:
diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp
@@ -1782,6 +1782,7 @@ void Driver::buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
       }
       case file_types::TY_SwiftModuleFile:
       case file_types::TY_SwiftModuleDocFile:
+      case file_types::TY_SwiftSourceInfoFile:
         if (OI.ShouldGenerateModule && !OI.shouldLink()) {
           // When generating a .swiftmodule as a top-level output (as opposed
           // to, for example, linking an image), treat .swiftmodule files as
@@ -2541,9 +2542,11 @@ Job *Driver::buildJobsForAction(Compilation &C, const JobAction *JA,
     chooseSwiftModuleOutputPath(C, OutputMap, workingDirectory, Output.get());
 
   if (OI.ShouldGenerateModule &&
-      (isa<CompileJobAction>(JA) || isa<MergeModuleJobAction>(JA)))
-    chooseSwiftModuleDocOutputPath(C, OutputMap, workingDirectory,
-                                   Output.get());
+      (isa<CompileJobAction>(JA) || isa<MergeModuleJobAction>(JA))) {
+    chooseSwiftModuleDocOutputPath(C, OutputMap, workingDirectory, Output.get());
+    choosePrivateOutputFilePath(C, OutputMap, workingDirectory, Output.get(),
+                                file_types::TY_SwiftSourceInfoFile);
+  }
 
   if (C.getArgs().hasArg(options::OPT_emit_module_interface,
                          options::OPT_emit_module_interface_path))
@@ -2786,6 +2789,41 @@ void Driver::chooseSwiftModuleOutputPath(Compilation &C,
   }
 }
 
+void Driver::choosePrivateOutputFilePath(Compilation &C,
+                                         const TypeToPathMap *OutputMap,
+                                         StringRef workingDirectory,
+                                         CommandOutput *Output,
+                                         file_types::ID fileID) const {
+  if (hasExistingAdditionalOutput(*Output, fileID))
+    return;
+
+  StringRef OFMOutputPath;
+  if (OutputMap) {
+    auto iter = OutputMap->find(fileID);
+    if (iter != OutputMap->end())
+      OFMOutputPath = iter->second;
+  }
+  if (!OFMOutputPath.empty()) {
+    // Prefer a path from the OutputMap.
+    Output->setAdditionalOutputForType(fileID, OFMOutputPath);
+  } else if (Output->getPrimaryOutputType() != file_types::TY_Nothing) {
+    auto ModulePath = Output->getAnyOutputForType(file_types::TY_SwiftModuleFile);
+    bool isTempFile = C.isTemporaryFile(ModulePath);
+    auto ModuleName = llvm::sys::path::filename(ModulePath);
+    llvm::SmallString<128> Path(llvm::sys::path::parent_path(ModulePath));
+    llvm::sys::path::append(Path, "Private");
+    // If the build system has created a Private dir for us to include the file, use it.
+    if (!llvm::sys::fs::exists(Path)) {
+      llvm::sys::path::remove_filename(Path);
+    }
+    llvm::sys::path::append(Path, ModuleName);
+    llvm::sys::path::replace_extension(Path, file_types::getExtension(fileID));
+    Output->setAdditionalOutputForType(fileID, Path);
+    if (isTempFile)
+      C.addTemporaryFile(Path);
+  }
+}
+
 void Driver::chooseSwiftModuleDocOutputPath(Compilation &C,
                                             const TypeToPathMap *OutputMap,
                                             StringRef workingDirectory,
diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp
@@ -531,6 +531,7 @@ const char *ToolChain::JobContext::computeFrontendModeForCompile() const {
   case file_types::TY_TBD:
   case file_types::TY_OptRecord:
   case file_types::TY_SwiftModuleInterfaceFile:
+  case file_types::TY_SwiftSourceInfoFile:
     llvm_unreachable("Output type can never be primary output.");
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID");
@@ -639,6 +640,9 @@ void ToolChain::JobContext::addFrontendSupplementaryOutputArguments(
   addOutputsOfType(arguments, Output, Args, file_types::TY_SwiftModuleDocFile,
                    "-emit-module-doc-path");
 
+  addOutputsOfType(arguments, Output, Args, file_types::TY_SwiftSourceInfoFile,
+                   "-emit-module-source-info-path");
+
   addOutputsOfType(arguments, Output, Args,
                    file_types::ID::TY_SwiftModuleInterfaceFile,
                    "-emit-module-interface-path");
@@ -770,6 +774,7 @@ ToolChain::constructInvocation(const BackendJobAction &job,
     case file_types::TY_ModuleTrace:
     case file_types::TY_OptRecord:
     case file_types::TY_SwiftModuleInterfaceFile:
+    case file_types::TY_SwiftSourceInfoFile:
       llvm_unreachable("Output type can never be primary output.");
     case file_types::TY_INVALID:
       llvm_unreachable("Invalid type ID");
@@ -907,6 +912,8 @@ ToolChain::constructInvocation(const MergeModuleJobAction &job,
 
   addOutputsOfType(Arguments, context.Output, context.Args,
                    file_types::TY_SwiftModuleDocFile, "-emit-module-doc-path");
+  addOutputsOfType(Arguments, context.Output, context.Args,
+                   file_types::TY_SwiftSourceInfoFile, "-emit-module-source-info-path");
   addOutputsOfType(Arguments, context.Output, context.Args,
                    file_types::ID::TY_SwiftModuleInterfaceFile,
                    "-emit-module-interface-path");
diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp
@@ -535,6 +535,12 @@ bool ArgsToFrontendOptionsConverter::checkUnusedSupplementaryOutputPaths()
     Diags.diagnose(SourceLoc(), diag::error_mode_cannot_emit_module_doc);
     return true;
   }
+  // If we cannot emit module doc, we cannot emit source information file either.
+  if (!FrontendOptions::canActionEmitModuleDoc(Opts.RequestedAction) &&
+      Opts.InputsAndOutputs.hasModuleSourceInfoOutputPath()) {
+     Diags.diagnose(SourceLoc(), diag::error_mode_cannot_emit_module_source_info);
+     return true;
+   }
   if (!FrontendOptions::canActionEmitInterface(Opts.RequestedAction) &&
       Opts.InputsAndOutputs.hasModuleInterfaceOutputPath()) {
     Diags.diagnose(SourceLoc(), diag::error_mode_cannot_emit_interface);
diff --git a/lib/Frontend/ArgsToFrontendOutputsConverter.cpp b/lib/Frontend/ArgsToFrontendOutputsConverter.cpp
@@ -296,11 +296,13 @@ SupplementaryOutputPathsComputer::getSupplementaryOutputPathsFromArguments()
   auto TBD = getSupplementaryFilenamesFromArguments(options::OPT_emit_tbd_path);
   auto moduleInterfaceOutput = getSupplementaryFilenamesFromArguments(
       options::OPT_emit_module_interface_path);
+  auto moduleSourceInfoOutput = getSupplementaryFilenamesFromArguments(
+      options::OPT_emit_module_source_info_path);
 
   if (!objCHeaderOutput || !moduleOutput || !moduleDocOutput ||
       !dependenciesFile || !referenceDependenciesFile ||
       !serializedDiagnostics || !fixItsOutput || !loadedModuleTrace || !TBD ||
-      !moduleInterfaceOutput) {
+      !moduleInterfaceOutput || !moduleSourceInfoOutput) {
     return None;
   }
   std::vector<SupplementaryOutputPaths> result;
@@ -319,7 +321,7 @@ SupplementaryOutputPathsComputer::getSupplementaryOutputPathsFromArguments()
     sop.LoadedModuleTracePath = (*loadedModuleTrace)[i];
     sop.TBDPath = (*TBD)[i];
     sop.ModuleInterfaceOutputPath = (*moduleInterfaceOutput)[i];
-
+    sop.ModuleSourceInfoOutputPath = (*moduleSourceInfoOutput)[i];
     result.push_back(sop);
   }
   return result;
@@ -394,6 +396,11 @@ SupplementaryOutputPathsComputer::computeOutputPathsForOneInput(
       file_types::TY_SwiftModuleDocFile, "",
       defaultSupplementaryOutputPathExcludingExtension);
 
+  auto moduleSourceInfoOutputPath = determineSupplementaryOutputFilename(
+      OPT_emit_module_source_info, pathsFromArguments.ModuleSourceInfoOutputPath,
+      file_types::TY_SwiftSourceInfoFile, "",
+      defaultSupplementaryOutputPathExcludingExtension);
+
   // There is no non-path form of -emit-interface-path
   auto ModuleInterfaceOutputPath =
       pathsFromArguments.ModuleInterfaceOutputPath;
@@ -420,6 +427,7 @@ SupplementaryOutputPathsComputer::computeOutputPathsForOneInput(
   sop.LoadedModuleTracePath = loadedModuleTracePath;
   sop.TBDPath = tbdPath;
   sop.ModuleInterfaceOutputPath = ModuleInterfaceOutputPath;
+  sop.ModuleSourceInfoOutputPath = moduleSourceInfoOutputPath;
   return sop;
 }
 
@@ -489,6 +497,7 @@ createFromTypeToPathMap(const TypeToPathMap *map) {
       {file_types::TY_ObjCHeader, paths.ObjCHeaderOutputPath},
       {file_types::TY_SwiftModuleFile, paths.ModuleOutputPath},
       {file_types::TY_SwiftModuleDocFile, paths.ModuleDocOutputPath},
+      {file_types::TY_SwiftSourceInfoFile, paths.ModuleSourceInfoOutputPath},
       {file_types::TY_Dependencies, paths.DependenciesFilePath},
       {file_types::TY_SwiftDeps, paths.ReferenceDependenciesFilePath},
       {file_types::TY_SerializedDiagnostics, paths.SerializedDiagnosticsPath},
@@ -516,6 +525,7 @@ SupplementaryOutputPathsComputer::readSupplementaryOutputFileMap() const {
         options::OPT_serialize_diagnostics_path,
         options::OPT_emit_loaded_module_trace_path,
         options::OPT_emit_module_interface_path,
+        options::OPT_emit_module_source_info_path,
         options::OPT_emit_tbd_path)) {
     Diags.diagnose(SourceLoc(),
                    diag::error_cannot_have_supplementary_outputs,
diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp
@@ -129,6 +129,7 @@ SerializationOptions CompilerInvocation::computeSerializationOptions(
   SerializationOptions serializationOpts;
   serializationOpts.OutputPath = outs.ModuleOutputPath.c_str();
   serializationOpts.DocOutputPath = outs.ModuleDocOutputPath.c_str();
+  serializationOpts.SourceInfoOutputPath = outs.ModuleSourceInfoOutputPath.c_str();
   serializationOpts.GroupInfoPath = opts.GroupInfoPath.c_str();
   if (opts.SerializeBridgingHeader && !outs.ModuleOutputPath.empty())
     serializationOpts.ImportedHeader = opts.ImplicitObjCHeaderPath;
diff --git a/lib/Frontend/FrontendInputsAndOutputs.cpp b/lib/Frontend/FrontendInputsAndOutputs.cpp
@@ -435,6 +435,12 @@ bool FrontendInputsAndOutputs::hasModuleDocOutputPath() const {
         return outs.ModuleDocOutputPath;
       });
 }
+bool FrontendInputsAndOutputs::hasModuleSourceInfoOutputPath() const {
+  return hasSupplementaryOutputPath(
+  [](const SupplementaryOutputPaths &outs) -> const std::string & {
+    return outs.ModuleSourceInfoOutputPath;
+  });
+}
 bool FrontendInputsAndOutputs::hasModuleInterfaceOutputPath() const {
   return hasSupplementaryOutputPath(
       [](const SupplementaryOutputPaths &outs) -> const std::string & {
diff --git a/lib/Frontend/FrontendOptions.cpp b/lib/Frontend/FrontendOptions.cpp
@@ -127,7 +127,8 @@ void FrontendOptions::forAllOutputPaths(
   const std::string *outputs[] = {&outs.ModuleOutputPath,
                                   &outs.ModuleDocOutputPath,
                                   &outs.ModuleInterfaceOutputPath,
-                                  &outs.ObjCHeaderOutputPath};
+                                  &outs.ObjCHeaderOutputPath,
+                                  &outs.ModuleSourceInfoOutputPath};
   for (const std::string *next : outputs) {
     if (!next->empty())
       fn(*next);
diff --git a/lib/Frontend/ModuleInterfaceBuilder.cpp b/lib/Frontend/ModuleInterfaceBuilder.cpp
@@ -366,7 +366,9 @@ bool ModuleInterfaceBuilder::buildSwiftModule(
       // We don't want to serialize module docs in the cache -- they
       // will be serialized beside the interface file.
       serializeToBuffers(Mod, SerializationOpts, ModuleBuffer,
-                         /*ModuleDocBuffer*/nullptr, SILMod.get());
+                         /*ModuleDocBuffer*/nullptr,
+                         /*SourceInfoBuffer*/nullptr,
+                         SILMod.get());
     });
 
     LLVM_DEBUG(llvm::dbgs() << "Running SIL processing passes\n");
diff --git a/lib/Serialization/Serialization.cpp b/lib/Serialization/Serialization.cpp
@@ -4738,6 +4738,7 @@ void swift::serializeToBuffers(
   ModuleOrSourceFile DC, const SerializationOptions &options,
   std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
   std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
+  std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
   const SILModule *M) {
 
   assert(!StringRef::withNullAsEmpty(options.OutputPath).empty());
@@ -4774,6 +4775,22 @@ void swift::serializeToBuffers(
       *moduleDocBuffer = llvm::make_unique<llvm::SmallVectorMemoryBuffer>(
                            std::move(buf), options.DocOutputPath);
   }
+
+  if (!StringRef::withNullAsEmpty(options.SourceInfoOutputPath).empty()) {
+    SharedTimer timer("Serialization, swiftsourceinfo, to buffer");
+    llvm::SmallString<1024> buf;
+    llvm::raw_svector_ostream stream(buf);
+    writeSourceInfoToStream(stream, DC);
+    (void)withOutputFile(getContext(DC).Diags,
+                         options.SourceInfoOutputPath,
+                         [&](raw_ostream &out) {
+      out << stream.str();
+      return false;
+    });
+    if (moduleSourceInfoBuffer)
+      *moduleSourceInfoBuffer = llvm::make_unique<llvm::SmallVectorMemoryBuffer>(
+        std::move(buf), options.SourceInfoOutputPath);
+  }
 }
 
 void swift::serialize(ModuleOrSourceFile DC,
@@ -4807,4 +4824,14 @@ void swift::serialize(ModuleOrSourceFile DC,
       return false;
     });
   }
+
+  if (!StringRef::withNullAsEmpty(options.SourceInfoOutputPath).empty()) {
+    (void)withOutputFile(getContext(DC).Diags,
+                         options.SourceInfoOutputPath,
+                         [&](raw_ostream &out) {
+      SharedTimer timer("Serialization, swiftsourceinfo");
+      writeSourceInfoToStream(out, DC);
+      return false;
+    });
+  }
 }
diff --git a/lib/Serialization/Serialization.h b/lib/Serialization/Serialization.h
diff --git a/lib/Serialization/SerializeDoc.cpp b/lib/Serialization/SerializeDoc.cpp
