[Macros] Track macro dependency separately in module trace

Macro plugins are not normal Swift modules, track them differently.
Add "swiftmacros" field to the JSON file.

rdar://118013482

Author: rintaro

include/swift/AST/ModuleLoader.h

@@ -106,15 +106,25 @@ class DependencyTracker {
         : path(std::move(Path)), fingerprint(std::move(FP)) {}
   };
 
-  inline static std::string getPath(const IncrementalDependency &id) {
-    return id.path;
+  struct MacroPluginDependency {
+    Identifier moduleName;
+    std::string path;
+
+    MacroPluginDependency(const Identifier &moduleName, const std::string &path)
+        : moduleName(moduleName), path(path) {}
+  };
+
+  template <typename Dep>
+  inline static const std::string getPath(const Dep &dep) {
+    return dep.path;
   }
-  typedef ArrayRefView<IncrementalDependency, std::string, getPath>
-      DependencyPathArrayRef;
+  template <typename Dep>
+  using PathArrayRefView = ArrayRefView<Dep, const std::string, getPath>;
 
   std::shared_ptr<clang::DependencyCollector> clangCollector;
   SmallVector<IncrementalDependency, 8> incrementalDeps;
   llvm::StringSet<> incrementalDepsUniquer;
+  llvm::SetVector<MacroPluginDependency> macroPluginDeps;
 
 public:
   explicit DependencyTracker(
@@ -134,6 +144,12 @@ class DependencyTracker {
   /// \p File is performed.
   void addIncrementalDependency(StringRef File, Fingerprint FP);
 
+  /// Adds a macro plugin dependency.
+  ///
+  /// No additional canonicalization or adulteration of the file path in
+  /// \p File is performed.
+  void addMacroPluginDependency(StringRef File, Identifier ModuleName);
+
   /// Fetches the list of dependencies.
   ArrayRef<std::string> getDependencies() const;
 
@@ -143,8 +159,19 @@ class DependencyTracker {
 
   /// A view of the paths of the dependencies known to have incremental swift
   /// dependency information embedded inside of them.
-  DependencyPathArrayRef getIncrementalDependencyPaths() const {
-    return DependencyPathArrayRef(getIncrementalDependencies());
+  PathArrayRefView<IncrementalDependency>
+  getIncrementalDependencyPaths() const {
+    return PathArrayRefView<IncrementalDependency>(
+        getIncrementalDependencies());
+  }
+
+  /// The list of macro plugin dependencies.
+  ArrayRef<MacroPluginDependency> getMacroPluginDependencies() const;
+
+  PathArrayRefView<MacroPluginDependency>
+  getMacroPluginDependencyPaths() const {
+    return PathArrayRefView<MacroPluginDependency>(
+        getMacroPluginDependencies());
   }
 
   /// Return the underlying clang::DependencyCollector that this
@@ -353,4 +380,28 @@ class ModuleLoader {
 
 } // namespace swift
 
+namespace llvm {
+template <>
+struct DenseMapInfo<swift::DependencyTracker::MacroPluginDependency> {
+  using MacroPluginDependency = swift::DependencyTracker::MacroPluginDependency;
+
+  static MacroPluginDependency getEmptyKey() {
+    return {DenseMapInfo<swift::Identifier>::getEmptyKey(), ""};
+  }
+
+  static MacroPluginDependency getTombstoneKey() {
+    return {DenseMapInfo<swift::Identifier>::getTombstoneKey(), ""};
+  }
+
+  static unsigned getHashValue(MacroPluginDependency Val) {
+    return hash_combine(Val.moduleName, Val.path);
+  }
+
+  static bool isEqual(const MacroPluginDependency &LHS,
+                      const MacroPluginDependency &RHS) {
+    return LHS.moduleName == RHS.moduleName && LHS.path == RHS.path;
+  }
+};
+} // namespace llvm
+
 #endif

include/swift/AST/PluginLoader.h

@@ -88,8 +88,9 @@ class PluginLoader {
   llvm::Expected<LoadedExecutablePlugin *>
   loadExecutablePlugin(llvm::StringRef path);
 
-  /// Add the specified path to the dependency tracker if needed.
-  void recordDependency(StringRef path);
+  /// Add the specified plugin associated with the module name to the dependency
+  /// tracker if needed.
+  void recordDependency(const PluginEntry &plugin, Identifier moduleName);
 };
 
 } // namespace swift

include/swift/Basic/Statistics.def

@@ -115,6 +115,7 @@ FRONTEND_STATISTIC(AST, NumASTBytesAllocated)
 /// AST context's dependency collector.
 FRONTEND_STATISTIC(AST, NumDependencies)
 FRONTEND_STATISTIC(AST, NumIncrementalDependencies)
+FRONTEND_STATISTIC(AST, NumMacroPluginDependencies)
 
 /// Number of top-level, dynamic, and member names referenced in this frontend
 /// job's source file, as tracked by the AST context's referenced-name tracker.

lib/AST/FrontendSourceFileDepGraphFactory.cpp

@@ -550,6 +550,9 @@ class ExternalDependencyEnumerator {
     for (StringRef s : depTracker.getDependencies()) {
       enumerateUse<NodeKind::externalDepend>(enumerator, s, llvm::None);
     }
+    for (const auto &dep : depTracker.getMacroPluginDependencies()) {
+      enumerateUse<NodeKind::externalDepend>(enumerator, dep.path, llvm::None);
+    }
   }
 
 private:

lib/AST/ModuleLoader.cpp

@@ -59,6 +59,11 @@ void DependencyTracker::addIncrementalDependency(StringRef File,
   }
 }
 
+void DependencyTracker::addMacroPluginDependency(StringRef File,
+                                                 Identifier ModuleName) {
+  macroPluginDeps.insert({ModuleName, std::string(File)});
+}
+
 ArrayRef<std::string>
 DependencyTracker::getDependencies() const {
   return clangCollector->getDependencies();
@@ -69,6 +74,11 @@ DependencyTracker::getIncrementalDependencies() const {
   return incrementalDeps;
 }
 
+ArrayRef<DependencyTracker::MacroPluginDependency>
+DependencyTracker::getMacroPluginDependencies() const {
+  return macroPluginDeps.getArrayRef();
+}
+
 std::shared_ptr<clang::DependencyCollector>
 DependencyTracker::getClangCollector() {
   return clangCollector;

lib/AST/PluginLoader.cpp

@@ -150,12 +150,14 @@ const PluginLoader::PluginEntry &
 PluginLoader::lookupPluginByModuleName(Identifier moduleName) {
   auto &map = getPluginMap();
   auto found = map.find(moduleName);
-  if (found != map.end()) {
-    return found->second;
-  } else {
+  if (found == map.end()) {
     static PluginEntry notFound{"", ""};
     return notFound;
   }
+
+  // Track the dependency.
+  recordDependency(found->second, moduleName);
+  return found->second;
 }
 
 llvm::Expected<LoadedLibraryPlugin *>
@@ -166,9 +168,6 @@ PluginLoader::loadLibraryPlugin(StringRef path) {
     return llvm::createStringError(err, err.message());
   }
 
-  // Track the dependency.
-  recordDependency(path);
-
   // Load the plugin.
   auto plugin = getRegistry()->loadLibraryPlugin(resolvedPath);
   if (!plugin) {
@@ -193,9 +192,6 @@ PluginLoader::loadExecutablePlugin(StringRef path) {
     return llvm::createStringError(err, err.message());
   }
 
-  // Track the dependency.
-  recordDependency(path);
-
   // Load the plugin.
   auto plugin =
       getRegistry()->loadExecutablePlugin(resolvedPath, disableSandbox);
@@ -213,7 +209,26 @@ PluginLoader::loadExecutablePlugin(StringRef path) {
   return plugin;
 }
 
-void PluginLoader::recordDependency(StringRef path) {
-  if (DepTracker)
-    DepTracker->addDependency(path, /*IsSystem=*/false);
+void PluginLoader::recordDependency(const PluginEntry &plugin,
+                                    Identifier moduleName) {
+  if (!DepTracker)
+    return;
+
+  // libraryPath: non-nil, executablePath: nil: in-process library plugin.
+  // libraryPath: non-nil, executablePath: non-nil: external library plugin.
+  // libraryPath: nil, executablePath: non-nil: executable plugin.
+  StringRef path =
+      !plugin.libraryPath.empty() ? plugin.libraryPath : plugin.executablePath;
+
+  // NOTE: We don't track plugin-server path as a dependency because it doesn't
+  // provide much value.
+
+  assert(!path.empty());
+  SmallString<128> resolvedPath;
+  auto fs = Ctx.SourceMgr.getFileSystem();
+  if (auto err = fs->getRealPath(path, resolvedPath)) {
+    return;
+  }
+
+  DepTracker->addMacroPluginDependency(resolvedPath, moduleName);
 }

lib/Frontend/ModuleInterfaceBuilder.cpp

@@ -110,6 +110,9 @@ bool ExplicitModuleInterfaceBuilder::collectDepsForSerialization(
   auto IncDeps =
       Instance.getDependencyTracker()->getIncrementalDependencyPaths();
   InitialDepNames.append(IncDeps.begin(), IncDeps.end());
+  auto MacroDeps =
+      Instance.getDependencyTracker()->getMacroPluginDependencyPaths();
+  InitialDepNames.append(MacroDeps.begin(), MacroDeps.end());
   InitialDepNames.push_back(interfacePath.str());
   for (const auto &extra : extraDependencies) {
     InitialDepNames.push_back(extra.str());

lib/FrontendTool/FrontendTool.cpp

@@ -311,6 +311,7 @@ static void countASTStats(UnifiedStatsReporter &Stats,
   if (auto *D = Instance.getDependencyTracker()) {
     C.NumDependencies = D->getDependencies().size();
     C.NumIncrementalDependencies = D->getIncrementalDependencies().size();
+    C.NumMacroPluginDependencies = D->getMacroPluginDependencies().size();
   }
 
   for (auto SF : Instance.getPrimarySourceFiles()) {

lib/FrontendTool/LoadedModuleTrace.cpp

@@ -46,12 +46,18 @@ struct SwiftModuleTraceInfo {
   bool SupportsLibraryEvolution;
 };
 
+struct SwiftMacroTraceInfo {
+  Identifier Name;
+  std::string Path;
+};
+
 struct LoadedModuleTraceFormat {
   static const unsigned CurrentVersion = 2;
   unsigned Version;
   Identifier Name;
   std::string Arch;
   std::vector<SwiftModuleTraceInfo> SwiftModules;
+  std::vector<SwiftMacroTraceInfo> SwiftMacros;
 };
 } // namespace
 
@@ -68,6 +74,15 @@ template <> struct ObjectTraits<SwiftModuleTraceInfo> {
   }
 };
 
+template <>
+struct ObjectTraits<SwiftMacroTraceInfo> {
+  static void mapping(Output &out, SwiftMacroTraceInfo &contents) {
+    StringRef name = contents.Name.str();
+    out.mapRequired("name", name);
+    out.mapRequired("path", contents.Path);
+  }
+};
+
 // Version notes:
 // 1. Keys: name, arch, swiftmodules
 // 2. New keys: version, swiftmodulesDetailedInfo
@@ -87,6 +102,8 @@ template <> struct ObjectTraits<LoadedModuleTraceFormat> {
     out.mapRequired("swiftmodules", moduleNames);
 
     out.mapRequired("swiftmodulesDetailedInfo", contents.SwiftModules);
+
+    out.mapRequired("swiftmacros", contents.SwiftMacros);
   }
 };
 } // namespace json
@@ -549,8 +566,7 @@ void ABIDependencyEvaluator::printABIExportMap(llvm::raw_ostream &os) const {
 // FIXME: Use the VFS instead of handling paths directly. We are particularly
 // sloppy about handling relative paths in the dependency tracker.
 static void computeSwiftModuleTraceInfo(
-    ASTContext &ctx,
-    const SmallPtrSetImpl<ModuleDecl *> &abiDependencies,
+    ASTContext &ctx, const SmallPtrSetImpl<ModuleDecl *> &abiDependencies,
     const llvm::DenseMap<StringRef, ModuleDecl *> &pathToModuleDecl,
     const DependencyTracker &depTracker, StringRef prebuiltCachePath,
     std::vector<SwiftModuleTraceInfo> &traceInfo) {
@@ -567,8 +583,8 @@ static void computeSwiftModuleTraceInfo(
   SmallVector<std::string, 16> dependencies{deps.begin(), deps.end()};
   auto incrDeps = depTracker.getIncrementalDependencyPaths();
   dependencies.append(incrDeps.begin(), incrDeps.end());
-  auto sharedLibraryExtRegex =
-      llvm::Regex("dylib|so|dll", llvm::Regex::IgnoreCase);
+  // NOTE: macro dependencies are handled differently.
+  // See 'computeSwiftMacroTraceInfo()'.
   for (const auto &depPath : dependencies) {
 
     // Decide if this is a swiftmodule based on the extension of the raw
@@ -580,10 +596,8 @@ static void computeSwiftModuleTraceInfo(
     auto isSwiftmodule = moduleFileType == file_types::TY_SwiftModuleFile;
     auto isSwiftinterface =
         moduleFileType == file_types::TY_SwiftModuleInterfaceFile;
-    auto isSharedLibrary =
-        sharedLibraryExtRegex.match(llvm::sys::path::extension(depPath));
 
-    if (!(isSwiftmodule || isSwiftinterface || isSharedLibrary))
+    if (!(isSwiftmodule || isSwiftinterface))
       continue;
 
     auto dep = pathToModuleDecl.find(depPath);
@@ -639,34 +653,6 @@ static void computeSwiftModuleTraceInfo(
       continue;
     }
 
-    // If we found a shared library, it must be a compiler plugin dependency.
-    if (isSharedLibrary) {
-      // Infer the module name by dropping the library prefix and extension.
-      // e.g "/path/to/lib/libPlugin.dylib" -> "Plugin"
-      auto moduleName = llvm::sys::path::stem(depPath);
-      #if !defined(_WIN32)
-      moduleName.consume_front("lib");
-      #endif
-
-      StringRef realDepPath =
-          fs::real_path(depPath, buffer, /*expand_tile*/ true)
-              ? StringRef(depPath)
-              : buffer.str();
-
-      traceInfo.push_back(
-          {/*Name=*/
-           ctx.getIdentifier(moduleName),
-           /*Path=*/
-           realDepPath.str(),
-           /*IsImportedDirectly=*/
-           false,
-           /*SupportsLibraryEvolution=*/
-           false});
-      buffer.clear();
-
-      continue;
-    }
-
     // Skip cached modules in the prebuilt cache. We will add the corresponding
     // swiftinterface from the SDK directly, but this isn't checked. :-/
     //
@@ -706,6 +692,22 @@ static void computeSwiftModuleTraceInfo(
             });
 }
 
+static void
+computeSwiftMacroTraceInfo(ASTContext &ctx, const DependencyTracker &depTracker,
+                           std::vector<SwiftMacroTraceInfo> &traceInfo) {
+  for (const auto &macroDep : depTracker.getMacroPluginDependencies()) {
+    traceInfo.push_back({macroDep.moduleName, macroDep.path});
+  }
+
+  // Again, almost a re-implementation of reversePathSortedFilenames :(.
+  std::sort(
+      traceInfo.begin(), traceInfo.end(),
+      [](const SwiftMacroTraceInfo &m1, const SwiftMacroTraceInfo &m2) -> bool {
+        return std::lexicographical_compare(m1.Path.rbegin(), m1.Path.rend(),
+                                            m2.Path.rbegin(), m2.Path.rend());
+      });
+}
+
 // [NOTE: Bailing-vs-crashing-in-trace-emission] There are certain edge cases
 // in trace emission where an invariant that you think should hold does not hold
 // in practice. For example, sometimes we have seen modules without any
@@ -765,10 +767,14 @@ bool swift::emitLoadedModuleTraceIfNeeded(ModuleDecl *mainModule,
                               *depTracker, opts.PrebuiltModuleCachePath,
                               swiftModules);
 
+  std::vector<SwiftMacroTraceInfo> swiftMacros;
+  computeSwiftMacroTraceInfo(ctxt, *depTracker, swiftMacros);
+
   LoadedModuleTraceFormat trace = {
       /*version=*/LoadedModuleTraceFormat::CurrentVersion,
       /*name=*/mainModule->getName(),
-      /*arch=*/ctxt.LangOpts.Target.getArchName().str(), swiftModules};
+      /*arch=*/ctxt.LangOpts.Target.getArchName().str(), swiftModules,
+      swiftMacros};
 
   // raw_fd_ostream is unbuffered, and we may have multiple processes writing,
   // so first write to memory and then dump the buffer to the trace file.

lib/FrontendTool/MakeStyleDependencies.cpp

@@ -132,6 +132,12 @@ bool swift::emitMakeDependenciesIfNeeded(DiagnosticEngine &diags,
     dependencyString.push_back(' ');
     dependencyString.append(frontend::utils::escapeForMake(path, buffer).str());
   }
+  auto macroPluginDependencyPath =
+      reversePathSortedFilenames(depTracker->getMacroPluginDependencyPaths());
+  for (auto const &path : macroPluginDependencyPath) {
+    dependencyString.push_back(' ');
+    dependencyString.append(frontend::utils::escapeForMake(path, buffer).str());
+  }
 
   // FIXME: Xcode can't currently handle multiple targets in a single
   // dependency line.