Merge pull request github#10956 from github/redsun82/swift-linkage-awareness

Swift: disambuigate entities using linkage awareness on modules

Author: redsun82

swift/extractor/SwiftExtractor.cpp

@@ -8,9 +8,8 @@
 #include <swift/AST/SourceFile.h>
 #include <swift/AST/Builtins.h>
 
-#include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/translators/SwiftVisitor.h"
-#include "swift/extractor/TargetTrapFile.h"
+#include "swift/extractor/infra/TargetDomains.h"
 #include "swift/extractor/SwiftBuiltinSymbols.h"
 #include "swift/extractor/infra/file/Path.h"
 
@@ -109,21 +108,24 @@ static llvm::SmallVector<swift::Decl*> getTopLevelDecls(swift::ModuleDecl& modul
 }
 
 static std::unordered_set<swift::ModuleDecl*> extractDeclarations(
-    const SwiftExtractorConfiguration& config,
+    SwiftExtractorState& state,
     swift::CompilerInstance& compiler,
     swift::ModuleDecl& module,
     swift::SourceFile* primaryFile = nullptr) {
   auto filename = getFilename(module, primaryFile);
+  if (primaryFile) {
+    state.sourceFiles.push_back(filename);
+  }
 
   // The extractor can be called several times from different processes with
   // the same input file(s). Using `TargetFile` the first process will win, and the following
   // will just skip the work
-  auto trapTarget = createTargetTrapFile(config, filename);
-  if (!trapTarget) {
+  const auto trapType = primaryFile ? TrapType::source : TrapType::module;
+  auto trap = createTargetTrapDomain(state, filename, trapType);
+  if (!trap) {
     // another process arrived first, nothing to do for us
     return {};
   }
-  TrapDomain trap{*trapTarget};
 
   std::vector<swift::Token> comments;
   if (primaryFile && primaryFile->getBufferID().hasValue()) {
@@ -137,7 +139,7 @@ static std::unordered_set<swift::ModuleDecl*> extractDeclarations(
     }
   }
 
-  SwiftVisitor visitor(compiler.getSourceMgr(), trap, module, primaryFile);
+  SwiftVisitor visitor(compiler.getSourceMgr(), *trap, module, primaryFile);
   auto topLevelDecls = getTopLevelDecls(module, primaryFile);
   for (auto decl : topLevelDecls) {
     visitor.extract(decl);
@@ -171,11 +173,10 @@ static std::vector<swift::ModuleDecl*> collectLoadedModules(swift::CompilerInsta
   return ret;
 }
 
-void codeql::extractSwiftFiles(const SwiftExtractorConfiguration& config,
-                               swift::CompilerInstance& compiler) {
+void codeql::extractSwiftFiles(SwiftExtractorState& state, swift::CompilerInstance& compiler) {
   auto inputFiles = collectInputFilenames(compiler);
   std::vector<swift::ModuleDecl*> todo = collectLoadedModules(compiler);
-  std::unordered_set<swift::ModuleDecl*> seen{todo.begin(), todo.end()};
+  state.encounteredModules.insert(todo.begin(), todo.end());
 
   while (!todo.empty()) {
     auto module = todo.back();
@@ -191,16 +192,16 @@ void codeql::extractSwiftFiles(const SwiftExtractorConfiguration& config,
       if (inputFiles.count(sourceFile->getFilename().str()) == 0) {
         continue;
       }
-      archiveFile(config, *sourceFile);
-      encounteredModules = extractDeclarations(config, compiler, *module, sourceFile);
+      archiveFile(state.configuration, *sourceFile);
+      encounteredModules = extractDeclarations(state, compiler, *module, sourceFile);
     }
     if (!isFromSourceFile) {
-      encounteredModules = extractDeclarations(config, compiler, *module);
+      encounteredModules = extractDeclarations(state, compiler, *module);
     }
     for (auto encountered : encounteredModules) {
-      if (seen.count(encountered) == 0) {
+      if (state.encounteredModules.count(encountered) == 0) {
         todo.push_back(encountered);
-        seen.insert(encountered);
+        state.encounteredModules.insert(encountered);
       }
     }
   }

swift/extractor/SwiftExtractor.h

@@ -1,11 +1,10 @@
 #pragma once
 
-#include "swift/extractor/config/SwiftExtractorConfiguration.h"
+#include "swift/extractor/config/SwiftExtractorState.h"
 #include <swift/AST/SourceFile.h>
 #include <swift/Frontend/Frontend.h>
 #include <memory>
 
 namespace codeql {
-void extractSwiftFiles(const SwiftExtractorConfiguration& config,
-                       swift::CompilerInstance& compiler);
+void extractSwiftFiles(SwiftExtractorState& state, swift::CompilerInstance& compiler);
 }  // namespace codeql

swift/extractor/TargetTrapFile.cpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <vector>
+#include <unordered_set>
+#include <filesystem>
+
+#include <swift/AST/Decl.h>
+
+#include "swift/extractor/config/SwiftExtractorConfiguration.h"
+
+namespace codeql {
+struct SwiftExtractorState {
+  const SwiftExtractorConfiguration configuration;
+
+  // All the trap files related to this extraction. This may also include trap files generated in a
+  // previous run but that this run requested as well. Paths are relative to `configuration.trapDir`
+  std::vector<std::filesystem::path> traps;
+
+  // All modules encountered during this extractor run, which therefore are dependencies of the
+  // outcomes of this run
+  std::unordered_set<const swift::ModuleDecl*> encounteredModules;
+
+  std::vector<std::filesystem::path> sourceFiles;
+
+  // The path for the modules outputted by the underlying frontend run, ignoring path redirection
+  std::vector<std::filesystem::path> originalOutputModules;
+};
+
+}  // namespace codeql

swift/extractor/TargetTrapFile.h

@@ -6,6 +6,7 @@ swift_cc_library(
     hdrs = glob(["*.h"]),
     visibility = ["//swift:__subpackages__"],
     deps = [
+        "//swift/extractor/config",
         "//swift/extractor/infra/file",
         "//swift/extractor/trap",
         "//swift/third_party/swift-llvm-support",

swift/extractor/config/SwiftExtractorState.h

@@ -0,0 +1,85 @@
+#include "swift/extractor/infra/TargetDomains.h"
+#include <iomanip>
+
+namespace codeql {
+
+static const char* typeToStr(TrapType type) {
+  switch (type) {
+    case TrapType::source:
+      return "sources";
+    case TrapType::module:
+      return "modules";
+    case TrapType::invocation:
+      return "invocations";
+    case TrapType::linkage:
+      return "linkage";
+    default:
+      return "";
+  }
+}
+
+static std::filesystem::path getRelativeTrapPath(const std::filesystem::path& target,
+                                                 TrapType type,
+                                                 const char* extension = ".trap") {
+  auto trap = typeToStr(type) / target.relative_path();
+  trap += extension;
+  return trap;
+}
+
+std::filesystem::path getTrapPath(const SwiftExtractorState& state,
+                                  const std::filesystem::path& target,
+                                  TrapType type) {
+  return state.configuration.trapDir / getRelativeTrapPath(target, type);
+}
+
+std::optional<TrapDomain> createTargetTrapDomain(SwiftExtractorState& state,
+                                                 const std::filesystem::path& target,
+                                                 TrapType type) {
+  if (target.empty()) {
+    return std::nullopt;
+  }
+  auto trap = getRelativeTrapPath(target, type);
+  auto ret =
+      TargetFile::create(trap, state.configuration.trapDir, state.configuration.getTempTrapDir());
+  state.traps.push_back(std::move(trap));
+  if (ret) {
+    *ret << "/* extractor-args:\n";
+    for (const auto& opt : state.configuration.frontendOptions) {
+      *ret << "  " << std::quoted(opt) << " \\\n";
+    }
+    *ret << "\n*/\n";
+    return TrapDomain{*std::move(ret)};
+  }
+  return std::nullopt;
+}
+
+namespace {
+template <typename Domain>
+std::optional<Domain> createTarget(const SwiftExtractorState& state,
+                                   const std::filesystem::path& target,
+                                   const char* extension) {
+  if (target.empty()) {
+    return std::nullopt;
+  }
+  auto file = getRelativeTrapPath(target, TrapType::linkage, extension);
+  auto ret =
+      TargetFile::create(file, state.configuration.trapDir, state.configuration.getTempTrapDir());
+  if (ret) {
+    return Domain{*std::move(ret)};
+  }
+  return std::nullopt;
+}
+
+}  // namespace
+
+std::optional<LinkDomain> createTargetLinkDomain(const SwiftExtractorState& state,
+                                                 const std::filesystem::path& target) {
+  return createTarget<LinkDomain>(state, target, ".link");
+}
+
+std::optional<ObjectDomain> createTargetObjectDomain(const SwiftExtractorState& state,
+                                                     const std::filesystem::path& target) {
+  return createTarget<ObjectDomain>(state, target, ".odep");
+}
+
+}  // namespace codeql

swift/extractor/infra/BUILD.bazel

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "swift/extractor/trap/TrapDomain.h"
+#include "swift/extractor/trap/LinkDomain.h"
+#include "swift/extractor/trap/ObjectDomain.h"
+#include "swift/extractor/config/SwiftExtractorState.h"
+
+namespace codeql {
+
+enum class TrapType {
+  source,
+  module,
+  invocation,
+  linkage,
+};
+
+std::filesystem::path getTrapPath(const SwiftExtractorState& state,
+                                  const std::filesystem::path& target,
+                                  TrapType type);
+
+std::optional<TrapDomain> createTargetTrapDomain(SwiftExtractorState& state,
+                                                 const std::filesystem::path& target,
+                                                 TrapType type);
+
+std::optional<LinkDomain> createTargetLinkDomain(const SwiftExtractorState& state,
+                                                 const std::filesystem::path& target);
+
+std::optional<ObjectDomain> createTargetObjectDomain(const SwiftExtractorState& state,
+                                                     const std::filesystem::path& target);
+
+}  // namespace codeql

swift/extractor/infra/TargetDomains.cpp

@@ -21,7 +21,7 @@ std::filesystem::path resolvePath(const std::filesystem::path& path) {
   std::filesystem::path ret = {};
   static const auto canonicalize = shouldCanonicalize();
   if (canonicalize) {
-    ret = std::filesystem::canonical(path, ec);
+    ret = std::filesystem::weakly_canonical(path, ec);
   } else {
     ret = std::filesystem::absolute(path, ec);
   }

swift/extractor/infra/TargetDomains.h

@@ -6,6 +6,8 @@ swift_cc_library(
     hdrs = glob(["*.h"]),
     visibility = ["//swift:__subpackages__"],
     deps = [
+        "//swift/extractor/config",
         "//swift/extractor/infra",
+        "//swift/extractor/remapping",
     ],
 )