Swift: consume and store compiler frontend diagnostics

Author: AlexDenisov

swift/extractor/BUILD.bazel

@@ -10,9 +10,11 @@ swift_cc_binary(
     visibility = ["//swift:__pkg__"],
     deps = [
         "//swift/extractor/infra",
+        "//swift/extractor/invocation",
         "//swift/extractor/remapping",
         "//swift/extractor/translators",
         "//swift/third_party/swift-llvm-support",
+        "@picosha2",
     ],
 )
 

swift/extractor/invocation/BUILD.bazel

@@ -0,0 +1,11 @@
+load("//swift:rules.bzl", "swift_cc_library")
+
+swift_cc_library(
+    name = "invocation",
+    srcs = glob(["*.cpp"]),
+    hdrs = glob(["*.h"]),
+    visibility = ["//swift:__subpackages__"],
+    deps = [
+        "//swift/third_party/swift-llvm-support",
+    ],
+)

swift/extractor/invocation/CodeQLDiagnosticsConsumer.cpp

@@ -0,0 +1,97 @@
+#include "swift/extractor/invocation/CodeQLDiagnosticsConsumer.h"
+#include "swift/extractor/trap/TrapDomain.h"
+#include "swift/extractor/trap/generated/TrapEntries.h"
+
+#include <swift/AST/DiagnosticEngine.h>
+#include <swift/Basic/SourceManager.h>
+#include <llvm/ADT/SmallString.h>
+#include <llvm/Support/raw_ostream.h>
+#include <string>
+
+using namespace codeql;
+
+static int diagnosticsKind(const swift::DiagnosticInfo& diagInfo) {
+  switch (diagInfo.Kind) {
+    case swift::DiagnosticKind::Error:
+      return 1;
+    case swift::DiagnosticKind::Warning:
+      return 2;
+
+    case swift::DiagnosticKind::Note:
+      return 3;
+
+    case swift::DiagnosticKind::Remark:
+      return 4;
+  }
+  return 0;
+}
+
+static std::filesystem::path getFilePath(std::string_view path) {
+  // TODO: this needs more testing
+  // TODO: check canonicalization of names on a case insensitive filesystems
+  // TODO: make symlink resolution conditional on CODEQL_PRESERVE_SYMLINKS=true
+  std::error_code ec;
+  auto ret = std::filesystem::canonical(path, ec);
+  if (ec) {
+    std::cerr << "Cannot get real path: " << std::quoted(path) << ": " << ec.message() << "\n";
+    return {};
+  }
+  return ret;
+}
+
+static void attachLocation(TrapDomain& trap,
+                           swift::SourceManager& sourceManager,
+                           const swift::DiagnosticInfo& diagInfo,
+                           DiagnosticsTrap& locatable) {
+  auto loc = diagInfo.Loc;
+  if (!loc.isValid()) {
+    return;
+  }
+  auto filepath = getFilePath(sourceManager.getDisplayNameForLoc(loc));
+  FilesTrap file;
+  file.id = trap.createLabel<FileTag>();
+  file.name = filepath;
+  trap.emit(file);
+
+  LocationsTrap location;
+  location.id = trap.createLabel<LocationTag>();
+  location.file = file.id;
+  std::tie(location.start_line, location.start_column) =
+      sourceManager.getLineAndColumnInBuffer(loc);
+  std::tie(location.end_line, location.end_column) = sourceManager.getLineAndColumnInBuffer(loc);
+  trap.emit(location);
+  trap.emit(LocatableLocationsTrap{locatable.id, location.id});
+}
+
+void CodeQLDiagnosticsConsumer::handleDiagnostic(swift::SourceManager& sourceManager,
+                                                 const swift::DiagnosticInfo& diagInfo) {
+  auto message = getDiagMessage(sourceManager, diagInfo);
+  DiagnosticsTrap diag{};
+  diag.id = trap.createLabel<DiagnosticsTag>();
+  diag.kind = diagnosticsKind(diagInfo);
+  diag.text = message;
+  trap.emit(diag);
+  attachLocation(trap, sourceManager, diagInfo, diag);
+}
+
+std::string CodeQLDiagnosticsConsumer::getDiagMessage(swift::SourceManager& sourceManager,
+                                                      const swift::DiagnosticInfo& diagInfo) {
+  // Translate ranges.
+  llvm::SmallVector<llvm::SMRange, 2> ranges;
+  for (auto R : diagInfo.Ranges)
+    ranges.push_back(getRawRange(sourceManager, R));
+
+  // Translate fix-its.
+  llvm::SmallVector<llvm::SMFixIt, 2> fixIts;
+  for (const swift::DiagnosticInfo::FixIt& F : diagInfo.FixIts)
+    fixIts.push_back(getRawFixIt(sourceManager, F));
+
+  // Actually substitute the diagnostic arguments into the diagnostic text.
+  llvm::SmallString<256> Text;
+  {
+    llvm::raw_svector_ostream Out(Text);
+    swift::DiagnosticEngine::formatDiagnosticText(Out, diagInfo.FormatString, diagInfo.FormatArgs);
+  }
+
+  return Text.str().str();
+}

swift/extractor/invocation/CodeQLDiagnosticsConsumer.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <swift/AST/DiagnosticConsumer.h>
+
+namespace codeql {
+
+class TrapDomain;
+
+class CodeQLDiagnosticsConsumer : public swift::DiagnosticConsumer {
+ public:
+  explicit CodeQLDiagnosticsConsumer(TrapDomain& targetFile) : trap(targetFile) {}
+  void handleDiagnostic(swift::SourceManager& sourceManager,
+                        const swift::DiagnosticInfo& diagInfo) override;
+
+ private:
+  static std::string getDiagMessage(swift::SourceManager& sourceManager,
+                                    const swift::DiagnosticInfo& diagInfo);
+  TrapDomain& trap;
+};
+
+}  // namespace codeql

swift/extractor/main.cpp

@@ -1,12 +1,11 @@
 #include <fstream>
 #include <iomanip>
 #include <stdlib.h>
-#include <unordered_set>
 #include <vector>
-#include <string>
 #include <iostream>
 #include <regex>
 #include <unistd.h>
+#include <picosha2.h>
 
 #include <swift/Basic/LLVMInitialize.h>
 #include <swift/FrontendTool/FrontendTool.h>
@@ -15,6 +14,8 @@
 #include "swift/extractor/TargetTrapFile.h"
 #include "swift/extractor/remapping/SwiftOutputRewrite.h"
 #include "swift/extractor/remapping/SwiftOpenInterception.h"
+#include "swift/extractor/invocation/CodeQLDiagnosticsConsumer.h"
+#include "swift/extractor/trap/TrapDomain.h"
 
 using namespace std::string_literals;
 
@@ -23,14 +24,21 @@ using namespace std::string_literals;
 // semantic analysis
 class Observer : public swift::FrontendObserver {
  public:
-  explicit Observer(const codeql::SwiftExtractorConfiguration& config) : config{config} {}
+  explicit Observer(const codeql::SwiftExtractorConfiguration& config,
+                    codeql::CodeQLDiagnosticsConsumer& diagConsumer)
+      : config{config}, diagConsumer{diagConsumer} {}
+
+  void configuredCompiler(swift::CompilerInstance& instance) override {
+    instance.addDiagnosticConsumer(&diagConsumer);
+  }
 
   void performedSemanticAnalysis(swift::CompilerInstance& compiler) override {
     codeql::extractSwiftFiles(config, compiler);
   }
 
  private:
   const codeql::SwiftExtractorConfiguration& config;
+  codeql::CodeQLDiagnosticsConsumer& diagConsumer;
 };
 
 static std::string getenv_or(const char* envvar, const std::string& def) {
@@ -96,6 +104,23 @@ static void checkWhetherToRunUnderTool(int argc, char* const* argv) {
   execvp(args[0], args.data());
 }
 
+// Creates a target file that should store per-invocation info, e.g. compilation args,
+// compilations, diagnostics, etc.
+codeql::TargetFile invocationTargetFile(codeql::SwiftExtractorConfiguration& configuration) {
+  auto hasher = picosha2::hash256_one_by_one();
+  for (auto& option : configuration.frontendOptions) {
+    hasher.process(std::begin(option), std::end(option));
+  }
+  hasher.finish();
+  auto target = "invocation-"s + get_hash_hex_string(hasher);
+  auto maybeFile = codeql::createTargetTrapFile(configuration, target);
+  if (!maybeFile) {
+    std::cerr << "Cannot create invocation trap file: " << target << "\n";
+    abort();
+  }
+  return std::move(maybeFile.value());
+}
+
 int main(int argc, char** argv) {
   checkWhetherToRunUnderTool(argc, argv);
 
@@ -131,7 +156,10 @@ int main(int argc, char** argv) {
     args.push_back(arg.c_str());
   }
 
-  Observer observer(configuration);
+  auto invocationTrapFile = invocationTargetFile(configuration);
+  codeql::TrapDomain invocationDomain(invocationTrapFile);
+  codeql::CodeQLDiagnosticsConsumer diagConsumer(invocationDomain);
+  Observer observer(configuration, diagConsumer);
   int frontend_rc = swift::performFrontend(args, "swift-extractor", (void*)main, &observer);
 
   codeql::finalizeRemapping(remapping);