Swift: use std::filesystem and picoSHA2

This replaces usages of `llvm::fs` and string manipulation with
`std::filesystem`, also replacing `std::string` with
`std::filesystem::path` where it made sense.

Moreover MD5 hashing used in macOS file remapping was replaced by
SHA256 hashing using a small header-only SHA256 C++ library with an
MIT license, https://github.com/okdshin/PicoSHA2.

File contents hashing was relocated to the newly created `file` library
for later planned reuse.

Author: redsun82

misc/bazel/workspace.bzl

@@ -34,6 +34,14 @@ def codeql_workspace(repository_name = "codeql"):
         sha256 = "9f2cdee6dcc2039d4c47d25ab5141fe0678ce6ed27ef482cab17fe9fa38a30ce",
     )
 
+    http_archive(
+        name = "picosha2",
+        url = "https://github.com/okdshin/PicoSHA2/archive/27fcf6979298949e8a462e16d09a0351c18fcaf2.zip",
+        strip_prefix = "PicoSHA2-27fcf6979298949e8a462e16d09a0351c18fcaf2",
+        build_file = "@%s//swift/third_party/picosha2:BUILD.picosha2.bazel" % repository_name,
+        sha256 = "d6647ca45a8b7bdaf027ecb68d041b22a899a0218b7206dee755c558a2725abb",
+    )
+
     maybe(
         repo_rule = http_archive,
         name = "rules_pkg",

swift/extractor/SwiftExtractor.cpp

@@ -15,39 +15,42 @@
 
 using namespace codeql;
 using namespace std::string_literals;
+namespace fs = std::filesystem;
 
-static void archiveFile(const SwiftExtractorConfiguration& config, swift::SourceFile& file) {
-  if (std::error_code ec = llvm::sys::fs::create_directories(config.trapDir)) {
-    std::cerr << "Cannot create TRAP directory: " << ec.message() << "\n";
-    return;
-  }
+static fs::path toPath(llvm::StringRef s) {
+  return {static_cast<std::string_view>(s)};
+}
 
-  if (std::error_code ec = llvm::sys::fs::create_directories(config.sourceArchiveDir)) {
-    std::cerr << "Cannot create source archive directory: " << ec.message() << "\n";
-    return;
+static void ensureDirectory(const char* label, const fs::path& dir) {
+  std::error_code ec;
+  fs::create_directories(dir, ec);
+  if (ec) {
+    std::cerr << "Cannot create " << label << " directory: " << ec.message() << "\n";
+    std::abort();
   }
+}
 
-  llvm::SmallString<PATH_MAX> srcFilePath(file.getFilename());
-  llvm::sys::fs::make_absolute(srcFilePath);
+static void archiveFile(const SwiftExtractorConfiguration& config, swift::SourceFile& file) {
+  ensureDirectory("TRAP", config.trapDir);
+  ensureDirectory("source archive", config.sourceArchiveDir);
 
-  llvm::SmallString<PATH_MAX> dstFilePath(config.sourceArchiveDir);
-  llvm::sys::path::append(dstFilePath, srcFilePath);
+  fs::path srcFilePath = fs::absolute(toPath(file.getFilename()));
+  auto dstFilePath = config.sourceArchiveDir;
+  dstFilePath += srcFilePath;
 
-  llvm::StringRef parent = llvm::sys::path::parent_path(dstFilePath);
-  if (std::error_code ec = llvm::sys::fs::create_directories(parent)) {
-    std::cerr << "Cannot create source archive destination directory '" << parent.str()
-              << "': " << ec.message() << "\n";
-    return;
-  }
+  ensureDirectory("source archive destination", dstFilePath.parent_path());
+
+  std::error_code ec;
+  fs::copy(srcFilePath, dstFilePath, ec);
 
-  if (std::error_code ec = llvm::sys::fs::copy_file(srcFilePath, dstFilePath)) {
-    std::cerr << "Cannot archive source file '" << srcFilePath.str().str() << "' -> '"
-              << dstFilePath.str().str() << "': " << ec.message() << "\n";
+  if (ec) {
+    std::cerr << "Cannot archive source file " << srcFilePath << " -> " << dstFilePath
+              << "': " << ec.message() << "\n";
     return;
   }
 }
 
-static std::string getFilename(swift::ModuleDecl& module, swift::SourceFile* primaryFile) {
+static fs::path getFilename(swift::ModuleDecl& module, swift::SourceFile* primaryFile) {
   if (primaryFile) {
     return primaryFile->getFilename().str();
   }
@@ -57,7 +60,8 @@ static std::string getFilename(swift::ModuleDecl& module, swift::SourceFile* pri
     // In this case we want to differentiate them
     // Moreover, pcm files may come from caches located in different directories, but are
     // unambiguously identified by the base file name, so we can discard the absolute directory
-    std::string filename = "/pcms/"s + llvm::sys::path::filename(module.getModuleFilename()).str();
+    fs::path filename = "/pcms";
+    filename /= toPath(module.getModuleFilename()).filename();
     filename += "-";
     filename += module.getName().str();
     return filename;
@@ -66,7 +70,7 @@ static std::string getFilename(swift::ModuleDecl& module, swift::SourceFile* pri
     // The Builtin module has an empty filename, let's fix that
     return "/__Builtin__";
   }
-  auto filename = module.getModuleFilename().str();
+  auto filename = toPath(module.getModuleFilename());
   // there is a special case of a module without an actual filename reporting `<imports>`: in this
   // case we want to avoid the `<>` characters, in case a dirty DB is imported on Windows
   if (filename == "<imports>") {

swift/extractor/SwiftExtractorConfiguration.h

@@ -2,18 +2,19 @@
 
 #include <string>
 #include <vector>
+#include <filesystem>
 
 #include "swift/extractor/infra/file/TargetFile.h"
 
 namespace codeql {
 struct SwiftExtractorConfiguration {
   // The location for storing TRAP files to be imported by CodeQL engine.
-  std::string trapDir;
+  std::filesystem::path trapDir;
   // The location for storing extracted source files.
-  std::string sourceArchiveDir;
+  std::filesystem::path sourceArchiveDir;
   // A temporary directory that exists during database creation, but is deleted once the DB is
   // finalized.
-  std::string scratchDir;
+  std::filesystem::path scratchDir;
 
   // The original arguments passed to the extractor. Used for debugging.
   std::vector<std::string> frontendOptions;
@@ -22,18 +23,20 @@ struct SwiftExtractorConfiguration {
 
   // A temporary directory that contains TRAP files before they are moved into their final
   // destination.
-  std::string getTempTrapDir() const { return scratchDir + "/swift-trap-temp"; }
+  std::filesystem::path getTempTrapDir() const { return scratchDir / "swift-trap-temp"; }
 
   // VFS (virtual file system) support.
   // A temporary directory that contains VFS files used during extraction.
-  std::string getVFSDir() const { return scratchDir + "/swift-vfs"; }
+  std::filesystem::path getVFSDir() const { return scratchDir / "swift-vfs"; }
 
   // A temporary directory that contains temp VFS files before they moved into VFSDir.
-  std::string getTempVFSDir() const { return scratchDir + "/swift-vfs-temp"; }
+  std::filesystem::path getTempVFSDir() const { return scratchDir / "swift-vfs-temp"; }
 
   // A temporary directory that contains build artifacts generated by the extractor during the
   // overall extraction process.
-  std::string getTempArtifactDir() const { return scratchDir + "/swift-extraction-artifacts"; }
+  std::filesystem::path getTempArtifactDir() const {
+    return scratchDir / "swift-extraction-artifacts";
+  }
 };
 
 }  // namespace codeql

swift/extractor/TargetTrapFile.cpp

@@ -2,8 +2,8 @@
 #include <iomanip>
 namespace codeql {
 std::optional<TargetFile> createTargetTrapFile(const SwiftExtractorConfiguration& configuration,
-                                               std::string_view target) {
-  std::string trap{target};
+                                               const std::filesystem::path& target) {
+  auto trap = target;
   trap += ".trap";
   auto ret = TargetFile::create(trap, configuration.trapDir, configuration.getTempTrapDir());
   if (ret) {

swift/extractor/TargetTrapFile.h

@@ -6,6 +6,6 @@
 namespace codeql {
 
 std::optional<TargetFile> createTargetTrapFile(const SwiftExtractorConfiguration& configuration,
-                                               std::string_view target);
+                                               const std::filesystem::path& target);
 
 }  // namespace codeql

swift/extractor/infra/FilePath.h

@@ -1,15 +1,16 @@
 #pragma once
 
+#include <filesystem>
+
 #include <swift/AST/SourceFile.h>
 #include <swift/Basic/SourceManager.h>
-#include <llvm/Support/FileSystem.h>
 #include <swift/Parse/Token.h>
 
 #include "swift/extractor/trap/TrapLabelStore.h"
 #include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/infra/SwiftTagTraits.h"
 #include "swift/extractor/trap/generated/TrapClasses.h"
-#include "swift/extractor/infra/FilePath.h"
+#include "swift/extractor/infra/file/PathHash.h"
 
 namespace codeql {
 
@@ -29,7 +30,7 @@ class SwiftDispatcher {
                                const swift::Pattern*,
                                const swift::TypeRepr*,
                                const swift::TypeBase*,
-                               FilePath>;
+                               std::filesystem::path>;
 
   template <typename E>
   static constexpr bool IsStorable = std::is_constructible_v<Store::Handle, const E&>;
@@ -306,16 +307,17 @@ class SwiftDispatcher {
     return false;
   }
 
-  static FilePath getFilePath(llvm::StringRef path) {
+  static std::filesystem::path getFilePath(std::string_view path) {
     // TODO: this needs more testing
     // TODO: check canonicalization of names on a case insensitive filesystems
     // TODO: make symlink resolution conditional on CODEQL_PRESERVE_SYMLINKS=true
-    llvm::SmallString<PATH_MAX> realPath;
-    if (std::error_code ec = llvm::sys::fs::real_path(path, realPath)) {
-      std::cerr << "Cannot get real path: '" << path.str() << "': " << ec.message() << "\n";
+    std::error_code ec;
+    auto ret = std::filesystem::canonical(path, ec);
+    if (ec) {
+      std::cerr << "Cannot get real path: " << std::quoted(path) << ": " << ec.message() << "\n";
       return {};
     }
-    return realPath.str().str();
+    return ret;
   }
 
   // TODO: for const correctness these should consistently be `const` (and maybe const references
@@ -331,9 +333,9 @@ class SwiftDispatcher {
   virtual void visit(swift::TypeRepr* typeRepr, swift::Type type) = 0;
   virtual void visit(swift::TypeBase* type) = 0;
 
-  void visit(const FilePath& file) {
-    auto entry = createEntry(file, file.path);
-    entry.name = file.path;
+  void visit(const std::filesystem::path& file) {
+    auto entry = createEntry(file, file.string());
+    entry.name = file.string();
     emit(entry);
   }
 

swift/extractor/infra/SwiftDispatcher.h

@@ -2,10 +2,10 @@
 
 // This file implements the mapping needed by the API defined in the TrapTagTraits.h, so that
 // TrapTagOf/TrapLabelOf provide the tags/labels for specific swift entity types.
+#include <filesystem>
 #include <swift/AST/ASTVisitor.h>
 #include "swift/extractor/trap/TrapTagTraits.h"
 #include "swift/extractor/trap/generated/TrapTags.h"
-#include "swift/extractor/infra/FilePath.h"
 
 namespace codeql {
 
@@ -68,7 +68,7 @@ MAP_TYPE_TO_TAG(swift::TypeBase, TypeTag);
 OVERRIDE_TAG(FuncDecl, ConcreteFuncDeclTag);
 OVERRIDE_TAG(VarDecl, ConcreteVarDeclTag);
 
-MAP_TYPE_TO_TAG(FilePath, DbFileTag);
+MAP_TYPE_TO_TAG(std::filesystem::path, DbFileTag);
 
 #undef MAP_TAG
 #undef MAP_SUBTAG

swift/extractor/infra/SwiftTagTraits.h

@@ -5,5 +5,5 @@ swift_cc_library(
     srcs = glob(["*.cpp"]),
     hdrs = glob(["*.h"]),
     visibility = ["//swift:__subpackages__"],
-    deps = ["//swift/third_party/swift-llvm-support"],
+    deps = ["@picosha2"],
 )

swift/extractor/infra/file/BUILD.bazel

@@ -0,0 +1,32 @@
+#include "swift/extractor/infra/file/FileHash.h"
+#include <iostream>
+#include <fstream>
+#include <unistd.h>
+#include <fcntl.h>
+#include <picosha2.h>
+
+namespace codeql {
+std::string hashFile(const std::filesystem::path& file) {
+  // using `open` instead of `std::ifstream` to reuse `hashFile(int)` below
+  if (auto fd = ::open(file.c_str(), O_RDONLY); fd >= 0) {
+    return hashFile(fd);
+  }
+  return "";
+}
+
+std::string hashFile(int fd) {
+  auto hasher = picosha2::hash256_one_by_one();
+  constexpr size_t bufferSize = 16 * 1024;
+  char buffer[bufferSize];
+  ssize_t bytesRead = 0;
+  while ((bytesRead = ::read(fd, buffer, bufferSize)) > 0) {
+    hasher.process(buffer, buffer + bytesRead);
+  }
+  ::close(fd);
+  if (bytesRead < 0) {
+    return "";
+  }
+  hasher.finish();
+  return get_hash_hex_string(hasher);
+}
+}  // namespace codeql