Swift: fix Linux compatibility problem and fetch resource dir from toolchain

This fetches the resource directory directly from the released
toolchains, allowing us to stop prebuilding and assembling them.
Moreover insertion of our resource directory is moved to the lua
tracing configuration (solving a `TODO`) and enhanced. Now all options
that start with the original resource directory (either explicit or
implied) are redirected to our resource directory.

This solves a problem where `-I <original resource dir>/some/path` was
passed to the extractor and did not work.

This works around the 5.9 linux compatibility problem by including the
`PackageDescription` swift modules in the in-dist toolchain. Copying the
toolchain and fixing the `-I` flag was not enough as for some reason
compilation of `PackageDescription.swiftinterface`  was causing a crash
in the SIL pass. We work around that by pre-compiling those modules
during the build and  including `.swiftmodule` files in the resource
directory.

TODO (apart from testing):
* the libraries included in the macOS toolchain are now fat (they were
  intel only before), occupying more space. We should see if we need to
  trim them down.
* there might be other swiftinterface files causing problems on linux
  lurking around...
* if we go with this, we can simplify and trim down the prebuilding we
  do leaving out the resource directory.

Author: redsun82

swift/extractor/main.cpp

@@ -177,23 +177,14 @@ codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state) {
   return std::move(maybeDomain.value());
 }
 
-codeql::SwiftExtractorConfiguration configure(int argc,
-                                              char** argv,
-                                              const std::string& resourceDir) {
+codeql::SwiftExtractorConfiguration configure(int argc, char** argv) {
   codeql::SwiftExtractorConfiguration configuration{};
   configuration.trapDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_TRAP_DIR", "extractor-out/trap/swift");
   configuration.sourceArchiveDir =
       getenv_or("CODEQL_EXTRACTOR_SWIFT_SOURCE_ARCHIVE_DIR", "extractor-out/src");
   configuration.scratchDir =
       getenv_or("CODEQL_EXTRACTOR_SWIFT_SCRATCH_DIR", "extractor-out/working");
   configuration.frontendOptions.assign(argv + 1, argv + argc);
-  // TODO: Should be moved to the tracer config
-  for (int i = 0; i < argc - 1; i++) {
-    if (std::string("-resource-dir") == configuration.frontendOptions[i]) {
-      configuration.frontendOptions[i + 1] = resourceDir.c_str();
-      break;
-    }
-  }
   return configuration;
 }
 
@@ -226,9 +217,7 @@ int main(int argc, char** argv, char** envp) {
   INITIALIZE_LLVM();
   initializeSwiftModules();
 
-  std::string resourceDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_ROOT", ".") + "/resource-dir/" +
-                            getenv_or("CODEQL_PLATFORM", ".");
-  const auto configuration = configure(argc, argv, resourceDir);
+  const auto configuration = configure(argc, argv);
   LOG_INFO("calling extractor with arguments \"{}\"", argDump(argc, argv));
   LOG_DEBUG("environment:\n{}\n", envDump(envp));
 

swift/third_party/BUILD.swift-llvm-support.bazel

@@ -41,12 +41,3 @@ pkg_files(
     strip_prefix = strip_prefix.from_pkg(),
     visibility = ["//visibility:public"],
 )
-
-pkg_files(
-    name = "swift-resource-dir",
-    srcs = glob([
-        "toolchain/lib/swift/**/*",
-    ]),
-    strip_prefix = "toolchain/lib/swift",
-    visibility = ["//visibility:public"],
-)

swift/third_party/BUILD.swift-toolchain-linux.bazel

@@ -0,0 +1,49 @@
+load("@rules_pkg//:mappings.bzl", "pkg_filegroup", "pkg_files")
+
+_strip_prefix = "usr/lib/swift"
+
+_pm_interfaces_glob = "usr/lib/swift/pm/**/*.swiftinterface"
+
+pkg_files(
+    name = "resource-dir-original",
+    srcs = glob(
+        ["usr/lib/swift/**/*"],
+        exclude = [_pm_interfaces_glob],
+    ),
+    strip_prefix = _strip_prefix,
+)
+
+# pre-compile package manager interface files as compiling them hits a mysterious crash during SIL
+_pm_interface_files = [
+    (
+        f[len(_strip_prefix) + 1:f.rindex("/")],
+        f[f.rindex("/") + 1:],
+        f[f.rindex("/") + 1:f.rindex(".")] + ".swiftmodule",
+    )
+    for f in glob([_pm_interfaces_glob])
+]
+
+[
+    [
+        genrule(
+            name = "compile-%s" % interface,
+            srcs = ["%s/%s/%s" % (_strip_prefix, dir, interface)],
+            outs = [module],
+            cmd = "$(location usr/bin/swift-frontend) -compile-module-from-interface $< -o $@ -I $$(dirname $<)",
+            local = True,
+            tools = ["usr/bin/swift-frontend"],
+        ),
+        pkg_files(
+            name = "pkg-%s" % module,
+            srcs = [":compile-%s" % interface],
+            prefix = dir,
+        ),
+    ]
+    for dir, interface, module in _pm_interface_files
+]
+
+pkg_filegroup(
+    name = "resource-dir",
+    srcs = [":resource-dir-original"] + [":pkg-%s" % module for _, _, module in _pm_interface_files],
+    visibility = ["//visibility:public"],
+)

swift/third_party/BUILD.swift-toolchain-macos.bazel

@@ -0,0 +1,12 @@
+load("@rules_pkg//:mappings.bzl", "pkg_files")
+
+_strip_prefix = "usr/lib/swift"
+
+pkg_files(
+    name = "resource-dir",
+    srcs = glob(
+        ["usr/lib/swift/**/*"],
+    ),
+    strip_prefix = _strip_prefix,
+    visibility = ["//visibility:public"],
+)

swift/third_party/load.bzl

@@ -16,6 +16,92 @@ _swift_arch_map = {
     "macOS-X64": "darwin_x86_64",
 }
 
+_swift_version = _swift_prebuilt_version.rpartition(".")[0]
+
+_toolchain_info = {
+    "linux": struct(
+        platform = "ubuntu2204",
+        suffix = "ubuntu22.04",
+        extension = "tar.gz",
+        sha = "bca015e9d727ca39385d7e5b5399f46302d54a02218d40d1c3063662ffc6b42f",
+    ),
+    "macos": struct(
+        platform = "xcode",
+        suffix = "osx",
+        extension = "pkg",
+        sha = "3cf7a4b2f3efcfcb4fef42b6588a7b1c54f7b0f2d0a479f41c3e1620b045f48e",
+    ),
+}
+
+def _get_toolchain_url(info):
+    return "https://download.swift.org/%s/%s/%s/%s-%s.%s" % (
+        _swift_version.lower(),
+        info.platform,
+        _swift_version,
+        _swift_version,
+        info.suffix,
+        info.extension,
+    )
+
+def _toolchains(workspace_name):
+    rules = {
+        "tar.gz": http_archive,
+        "pkg": _pkg_archive,
+    }
+    for arch, info in _toolchain_info.items():
+        rule = rules[info.extension]
+        rule(
+            name = "swift_toolchain_%s" % arch,
+            url = _get_toolchain_url(info),
+            sha256 = info.sha,
+            build_file = _build(workspace_name, "swift-toolchain-%s" % arch),
+            strip_prefix = "%s-%s" % (_swift_version, info.suffix),
+        )
+
+def _run(repository_ctx, message, cmd, working_directory = "."):
+    repository_ctx.report_progress(message)
+    res = repository_ctx.execute(
+        ["bash", "-c", cmd],
+        working_directory = working_directory,
+    )
+    if res.return_code != 0:
+        fail(message)
+
+def _pkg_archive_impl(repository_ctx):
+    archive = "file.pkg"
+    url = repository_ctx.attr.url
+    dir = "%s-package.pkg" % repository_ctx.attr.strip_prefix
+    repository_ctx.report_progress("downloading %s" % url)
+    res = repository_ctx.download(
+        url,
+        output = archive,
+        sha256 = repository_ctx.attr.sha256,
+    )
+    if not repository_ctx.attr.sha256:
+        print("please set sha256 = %s" % repr(res.sha256))
+    _run(repository_ctx, "extracting %s" % dir, "xar -xf %s" % archive)
+    repository_ctx.delete(archive)
+    _run(
+        repository_ctx,
+        "extracting Payload from %s" % dir,
+        "cat %s/Payload | gunzip -dc | cpio -i" % dir,
+    )
+    repository_ctx.delete(dir)
+    repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD")
+    repository_ctx.file("WORKSPACE")
+
+_pkg_archive = repository_rule(
+    implementation = _pkg_archive_impl,
+    attrs = {
+        "url": attr.string(mandatory = True),
+        "sha256": attr.string(),
+        "strip_prefix": attr.string(),
+        "build_file": attr.label(mandatory = True),
+    },
+)
+
+# TODO apply the same mechanism to the macOS toolchain (needs some work as the toolchain is a pkg archive
+
 def _github_archive(*, name, repository, commit, build_file = None, sha256 = None):
     github_name = repository[repository.index("/") + 1:]
     maybe(
@@ -53,6 +139,8 @@ def load_dependencies(workspace_name):
             ],
         )
 
+    _toolchains(workspace_name)
+
     _github_archive(
         name = "picosha2",
         build_file = _build(workspace_name, "picosha2"),

swift/third_party/swift-llvm-support/BUILD.bazel

@@ -1,21 +1,25 @@
 package(default_visibility = ["//swift:__subpackages__"])
 
-#TODO we will be introducing universal binaries at a later stage, when we have both architectures prebuilt for macOS
-# for the moment, we make arm build an x86_64 binary
-_arch_override = {
-    "darwin_arm64": "darwin_x86_64",
-}
+alias(
+    name = "swift-llvm-support",
+    actual = select({
+        "@bazel_tools//src/conditions:linux": "@swift_prebuilt_linux//:swift-llvm-support",
+        "@bazel_tools//src/conditions:darwin": "@swift_prebuilt_darwin_x86_64//:swift-llvm-support",
+    }),
+)
 
-[
-    alias(
-        name = name,
-        actual = select({
-            "@bazel_tools//src/conditions:%s" % arch: "@swift_prebuilt_%s//:%s" % (
-                _arch_override.get(arch, arch),
-                name,
-            )
-            for arch in ("linux", "darwin_x86_64", "darwin_arm64")
-        }),
-    )
-    for name in ("swift-llvm-support", "swift-test-sdk", "swift-resource-dir")
-]
+alias(
+    name = "swift-test-sdk",
+    actual = select({
+        "@bazel_tools//src/conditions:linux": "@swift_prebuilt_linux//:swift-test-sdk",
+        "@bazel_tools//src/conditions:darwin": "@swift_prebuilt_darwin_x86_64//:swift-test-sdk",
+    }),
+)
+
+alias(
+    name = "swift-resource-dir",
+    actual = select({
+        "@bazel_tools//src/conditions:linux": "@swift_toolchain_linux//:resource-dir",
+        "@bazel_tools//src/conditions:darwin": "@swift_toolchain_macos//:resource-dir",
+    }),
+)