Merge pull request github#14629 from github/redsun82/rework-resource-dir

Swift: rework resource dir

Author: redsun82

swift/extractor/main.cpp

@@ -177,23 +177,14 @@ codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state) {
   return std::move(maybeDomain.value());
 }
 
-codeql::SwiftExtractorConfiguration configure(int argc,
-                                              char** argv,
-                                              const std::string& resourceDir) {
+codeql::SwiftExtractorConfiguration configure(int argc, char** argv) {
   codeql::SwiftExtractorConfiguration configuration{};
   configuration.trapDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_TRAP_DIR", "extractor-out/trap/swift");
   configuration.sourceArchiveDir =
       getenv_or("CODEQL_EXTRACTOR_SWIFT_SOURCE_ARCHIVE_DIR", "extractor-out/src");
   configuration.scratchDir =
       getenv_or("CODEQL_EXTRACTOR_SWIFT_SCRATCH_DIR", "extractor-out/working");
   configuration.frontendOptions.assign(argv + 1, argv + argc);
-  // TODO: Should be moved to the tracer config
-  for (int i = 0; i < argc - 1; i++) {
-    if (std::string("-resource-dir") == configuration.frontendOptions[i]) {
-      configuration.frontendOptions[i + 1] = resourceDir.c_str();
-      break;
-    }
-  }
   return configuration;
 }
 
@@ -226,9 +217,7 @@ int main(int argc, char** argv, char** envp) {
   INITIALIZE_LLVM();
   initializeSwiftModules();
 
-  std::string resourceDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_ROOT", ".") + "/resource-dir/" +
-                            getenv_or("CODEQL_PLATFORM", ".");
-  const auto configuration = configure(argc, argv, resourceDir);
+  const auto configuration = configure(argc, argv);
   LOG_INFO("calling extractor with arguments \"{}\"", argDump(argc, argv));
   LOG_DEBUG("environment:\n{}\n", envDump(envp));
 

swift/third_party/BUILD.swift-llvm-support.bazel

@@ -41,12 +41,3 @@ pkg_files(
     strip_prefix = strip_prefix.from_pkg(),
     visibility = ["//visibility:public"],
 )
-
-pkg_files(
-    name = "swift-resource-dir",
-    srcs = glob([
-        "toolchain/lib/swift/**/*",
-    ]),
-    strip_prefix = "toolchain/lib/swift",
-    visibility = ["//visibility:public"],
-)

swift/third_party/BUILD.swift-toolchain-linux.bazel

@@ -0,0 +1,49 @@
+load("@rules_pkg//:mappings.bzl", "pkg_filegroup", "pkg_files")
+
+_strip_prefix = "usr/lib/swift"
+
+_pm_interfaces_glob = "usr/lib/swift/pm/**/*.swiftinterface"
+
+pkg_files(
+    name = "resource-dir-original",
+    srcs = glob(
+        ["usr/lib/swift/**/*"],
+        exclude = [_pm_interfaces_glob],
+    ),
+    strip_prefix = _strip_prefix,
+)
+
+# pre-compile package manager interface files as compiling them hits a mysterious crash during SIL
+_pm_interface_files = [
+    (
+        f[len(_strip_prefix) + 1:f.rindex("/")],
+        f[f.rindex("/") + 1:],
+        f[f.rindex("/") + 1:f.rindex(".")] + ".swiftmodule",
+    )
+    for f in glob([_pm_interfaces_glob])
+]
+
+[
+    [
+        genrule(
+            name = "compile-%s" % interface,
+            srcs = ["%s/%s/%s" % (_strip_prefix, dir, interface)],
+            outs = [module],
+            cmd = "$(location usr/bin/swift-frontend) -compile-module-from-interface $< -o $@ -I $$(dirname $<)",
+            local = True,
+            tools = ["usr/bin/swift-frontend"],
+        ),
+        pkg_files(
+            name = "pkg-%s" % module,
+            srcs = [":compile-%s" % interface],
+            prefix = dir,
+        ),
+    ]
+    for dir, interface, module in _pm_interface_files
+]
+
+pkg_filegroup(
+    name = "resource-dir",
+    srcs = [":resource-dir-original"] + [":pkg-%s" % module for _, _, module in _pm_interface_files],
+    visibility = ["//visibility:public"],
+)

swift/third_party/BUILD.swift-toolchain-macos.bazel

@@ -0,0 +1,12 @@
+load("@rules_pkg//:mappings.bzl", "pkg_files")
+
+_strip_prefix = "usr/lib/swift"
+
+pkg_files(
+    name = "resource-dir",
+    srcs = glob(
+        ["usr/lib/swift/**/*"],
+    ),
+    strip_prefix = _strip_prefix,
+    visibility = ["//visibility:public"],
+)

swift/third_party/load.bzl

@@ -16,6 +16,91 @@ _swift_arch_map = {
     "macOS-X64": "darwin_x86_64",
 }
 
+_swift_version = _swift_prebuilt_version.rpartition(".")[0]
+
+_toolchain_info = {
+    "linux": struct(
+        platform = "ubuntu2004",
+        suffix = "ubuntu20.04",
+        extension = "tar.gz",
+        sha = "dd4b04c7f95c4ada4a2aacb66864b1ed20358313aaa4c880dc3974bf1eefa275",
+    ),
+    "macos": struct(
+        platform = "xcode",
+        suffix = "osx",
+        extension = "pkg",
+        sha = "417d46f73b2e6b5da82ebbc8a5f4979f7187691fd42119157ba56d5a8bc89eda",
+    ),
+}
+
+def _get_toolchain_url(info):
+    return "https://download.swift.org/%s/%s/%s/%s-%s.%s" % (
+        _swift_version.lower(),
+        info.platform,
+        _swift_version,
+        _swift_version,
+        info.suffix,
+        info.extension,
+    )
+
+def _toolchains(workspace_name):
+    rules = {
+        "tar.gz": http_archive,
+        "pkg": _pkg_archive,
+    }
+    for arch, info in _toolchain_info.items():
+        rule = rules[info.extension]
+        rule(
+            name = "swift_toolchain_%s" % arch,
+            url = _get_toolchain_url(info),
+            sha256 = info.sha,
+            build_file = _build(workspace_name, "swift-toolchain-%s" % arch),
+            strip_prefix = "%s-%s" % (_swift_version, info.suffix),
+        )
+
+def _run(repository_ctx, message, cmd, working_directory = "."):
+    repository_ctx.report_progress(message)
+    res = repository_ctx.execute(
+        ["bash", "-c", cmd],
+        working_directory = working_directory,
+    )
+    if res.return_code != 0:
+        fail(message)
+
+def _pkg_archive_impl(repository_ctx):
+    archive = "file.pkg"
+    url = repository_ctx.attr.url
+    dir = "%s-package.pkg" % repository_ctx.attr.strip_prefix
+    repository_ctx.report_progress("downloading %s" % url)
+    res = repository_ctx.download(
+        url,
+        output = archive,
+        sha256 = repository_ctx.attr.sha256,
+    )
+    if not repository_ctx.attr.sha256:
+        print("Rule '%s' indicated that a canonical reproducible form " % repository_ctx.name +
+              "can be obtained by modifying arguments sha256 = \"%s\"" % res.sha256)
+    _run(repository_ctx, "extracting %s" % dir, "xar -xf %s" % archive)
+    repository_ctx.delete(archive)
+    _run(
+        repository_ctx,
+        "extracting Payload from %s" % dir,
+        "cat %s/Payload | gunzip -dc | cpio -i" % dir,
+    )
+    repository_ctx.delete(dir)
+    repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD")
+    repository_ctx.file("WORKSPACE")
+
+_pkg_archive = repository_rule(
+    implementation = _pkg_archive_impl,
+    attrs = {
+        "url": attr.string(mandatory = True),
+        "sha256": attr.string(),
+        "strip_prefix": attr.string(),
+        "build_file": attr.label(mandatory = True),
+    },
+)
+
 def _github_archive(*, name, repository, commit, build_file = None, sha256 = None):
     github_name = repository[repository.index("/") + 1:]
     maybe(
@@ -53,6 +138,8 @@ def load_dependencies(workspace_name):
             ],
         )
 
+    _toolchains(workspace_name)
+
     _github_archive(
         name = "picosha2",
         build_file = _build(workspace_name, "picosha2"),

swift/third_party/swift-llvm-support/BUILD.bazel

@@ -1,21 +1,25 @@
 package(default_visibility = ["//swift:__subpackages__"])
 
-#TODO we will be introducing universal binaries at a later stage, when we have both architectures prebuilt for macOS
-# for the moment, we make arm build an x86_64 binary
-_arch_override = {
-    "darwin_arm64": "darwin_x86_64",
-}
+alias(
+    name = "swift-llvm-support",
+    actual = select({
+        "@bazel_tools//src/conditions:linux": "@swift_prebuilt_linux//:swift-llvm-support",
+        "@bazel_tools//src/conditions:darwin": "@swift_prebuilt_darwin_x86_64//:swift-llvm-support",
+    }),
+)
 
-[
-    alias(
-        name = name,
-        actual = select({
-            "@bazel_tools//src/conditions:%s" % arch: "@swift_prebuilt_%s//:%s" % (
-                _arch_override.get(arch, arch),
-                name,
-            )
-            for arch in ("linux", "darwin_x86_64", "darwin_arm64")
-        }),
-    )
-    for name in ("swift-llvm-support", "swift-test-sdk", "swift-resource-dir")
-]
+alias(
+    name = "swift-test-sdk",
+    actual = select({
+        "@bazel_tools//src/conditions:linux": "@swift_prebuilt_linux//:swift-test-sdk",
+        "@bazel_tools//src/conditions:darwin": "@swift_prebuilt_darwin_x86_64//:swift-test-sdk",
+    }),
+)
+
+alias(
+    name = "swift-resource-dir",
+    actual = select({
+        "@bazel_tools//src/conditions:linux": "@swift_toolchain_linux//:resource-dir",
+        "@bazel_tools//src/conditions:darwin": "@swift_toolchain_macos//:resource-dir",
+    }),
+)