Use emsdk as external bazel dependency (#766)

* Makes provided bazel rules look up @emsdk workspace instead of local workspace
* Uses system-specific emscripten binaries instead of defaulting to linux
* Provides macros for loading emsdk dependencies (nodejs and emscripten binaries)
* Unhardcodes paths in bazel rules and .sh wrappers
* `update_bazel_workspace.sh` now updates `revisions.bzl`
* `emscripten_deps()` can be fed with specific emscripten version
* Adds external usage test

Addresses #650 and #696

Author: celentes

.circleci/config.yml

@@ -166,7 +166,7 @@ jobs:
             docker login -u "$DOCKER_USER" -p "$DOCKER_PASS"
             make -C ./docker version=${CIRCLE_TAG} alias=latest push
 
-  test-bazel:
+  test-bazel-linux:
     executor: bionic
     steps:
       - checkout
@@ -186,6 +186,21 @@ jobs:
             apt-get install -q -y bazel
       - run: test/test_bazel.sh
 
+  test-bazel-mac:
+    macos:
+      xcode: "12.2.0"
+    environment:
+      EMSDK_NOTTY: "1"
+      HOMEBREW_NO_AUTO_UPDATE: "1"
+    steps:
+      - checkout
+      - run: brew install grep
+      - run:
+          name: install bazel
+          command: |
+            brew install bazel
+      - run: test/test_bazel_mac.sh
+
 workflows:
   flake8:
     jobs:
@@ -208,6 +223,9 @@ workflows:
               ignore: /.*/
             tags:
               only: /.*/
-  test-bazel:
+  test-bazel-linux:
+    jobs:
+      - test-bazel-linux
+  test-bazel-mac:
     jobs:
-      - test-bazel
+      - test-bazel-mac

bazel/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ['//visibility:public'])
+
+config_setting(
+    name = "linux",
+    constraint_values = [
+        "@platforms//os:linux",
+        "@platforms//cpu:x86_64",
+    ],
+)
+
+config_setting(
+    name = "macos",
+    constraint_values = [
+        "@platforms//os:macos",
+        "@platforms//cpu:x86_64",
+    ],
+)
+
+config_setting(
+    name = "windows",
+    constraint_values = [
+        "@platforms//os:windows",
+        "@platforms//cpu:x86_64",
+    ],
+)
+
+alias(
+    name = "binaries",
+    actual = select({
+        ":linux": "@emscripten_bin_linux//:all",
+        ":macos": "@emscripten_bin_mac//:all",
+        ":windows": "@emscripten_bin_win//:all",
+    }),
+)
+
+alias(
+    name = "node_modules",
+    actual = select({
+        ":linux": "@emscripten_npm_linux//:node_modules",
+        ":macos": "@emscripten_npm_mac//:node_modules",
+        ":windows": "@emscripten_npm_win//:node_modules",
+    }),
+)
+

bazel/WORKSPACE

@@ -1,25 +1,7 @@
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+workspace(name = "emsdk")
 
-http_archive(
-    name = "build_bazel_rules_nodejs",
-    sha256 = "0f2de53628e848c1691e5729b515022f5a77369c76a09fbe55611e12731c90e3",
-    urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/2.0.1/rules_nodejs-2.0.1.tar.gz"],
-)
+load(":deps.bzl", "deps")
+deps()
 
-load("@build_bazel_rules_nodejs//:index.bzl", "npm_install")
-
-# emscripten 2.0.15
-http_archive(
-    name = "emscripten",
-    sha256 = "7ff49fc63adf29970f6e7af1df445d7f554bdbbb2606db1cb5d3567ce69df1db",
-    strip_prefix = "install",
-    url = "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/89202930a98fe7f9ed59b574469a9471b0bda7dd/wasm-binaries.tbz2",
-    build_file = "//emscripten_toolchain:emscripten.BUILD",
-    type = "tar.bz2",
-)
-
-npm_install(
-    name = "npm",
-    package_json = "@emscripten//:emscripten/package.json",
-    package_lock_json = "@emscripten//:emscripten/package-lock.json",
-)
+load(":emscripten_deps.bzl", "emscripten_deps")
+emscripten_deps()

bazel/deps.bzl

@@ -0,0 +1,11 @@
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+def deps():
+    excludes = native.existing_rules().keys()
+
+    if "build_bazel_rules_nodejs" not in excludes:
+        http_archive(
+            name = "build_bazel_rules_nodejs",
+            sha256 = "0f2de53628e848c1691e5729b515022f5a77369c76a09fbe55611e12731c90e3",
+            urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/2.0.1/rules_nodejs-2.0.1.tar.gz"],
+        )

bazel/emscripten_deps.bzl

@@ -0,0 +1,76 @@
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@build_bazel_rules_nodejs//:index.bzl", "npm_install")
+load(":revisions.bzl", "EMSCRIPTEN_TAGS")
+
+def _parse_version(v):
+  return [int(u) for u in v.split(".")]
+
+def emscripten_deps(emscripten_version = "latest"):
+    version = emscripten_version
+
+    if version == "latest":
+        version = reversed(sorted(EMSCRIPTEN_TAGS.keys(), key=_parse_version))[0]
+
+    if version not in EMSCRIPTEN_TAGS.keys():
+        error_msg = "Emscripten version {} not found.".format(version)
+        error_msg += " Look at @emsdk//:revisions.bzl for the list "
+        error_msg += "of currently supported versions."
+        fail(error_msg)
+
+    revision = EMSCRIPTEN_TAGS[version]
+
+    emscripten_url = "https://storage.googleapis.com/webassembly/emscripten-releases-builds/{}/{}/wasm-binaries.tbz2"
+
+    # This could potentially backfire for projects with multiple emscripten
+    # dependencies that use different emscripten versions
+    excludes = native.existing_rules().keys()
+    if "emscripten_bin_linux" not in excludes:
+        http_archive(
+            name = "emscripten_bin_linux",
+            strip_prefix = "install",
+            url = emscripten_url.format("linux", revision.hash),
+            sha256 = revision.sha_linux,
+            build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD",
+            type = "tar.bz2",
+        )
+
+    if "emscripten_bin_mac" not in excludes:
+        http_archive(
+            name = "emscripten_bin_mac",
+            strip_prefix = "install",
+            url = emscripten_url.format("mac", revision.hash),
+            sha256 = revision.sha_mac,
+            build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD",
+            type = "tar.bz2",
+        )
+
+    if "emscripten_bin_win" not in excludes:
+        http_archive(
+            name = "emscripten_bin_win",
+            strip_prefix = "install",
+            url = emscripten_url.format("win", revision.hash),
+            sha256 = revision.sha_win,
+            build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD",
+            type = "tar.bz2",
+        )
+
+    if "emscripten_npm_linux" not in excludes:
+        npm_install(
+            name = "emscripten_npm_linux",
+            package_json = "@emscripten_bin_linux//:emscripten/package.json",
+            package_lock_json = "@emscripten_bin_linux//:emscripten/package-lock.json",
+        )
+
+    if "emscripten_npm_mac" not in excludes:
+        npm_install(
+            name = "emscripten_npm_mac",
+            package_json = "@emscripten_bin_mac//:emscripten/package.json",
+            package_lock_json = "@emscripten_bin_mac//:emscripten/package-lock.json",
+        )
+
+    if "emscripten_npm_win" not in excludes:
+        npm_install(
+            name = "emscripten_npm_win",
+            package_json = "@emscripten_bin_win//:emscripten/package.json",
+            package_lock_json = "@emscripten_bin_win//:emscripten/package-lock.json",
+        )

bazel/emscripten_toolchain/BUILD.bazel

@@ -9,9 +9,9 @@ filegroup(
         "emcc.sh",
         "emscripten_config",
         "env.sh",
-        "@emscripten//:all",
         "@nodejs//:node_files",
-        "@npm//:node_modules",
+        "@emsdk//:binaries",
+        "@emsdk//:node_modules",
     ],
 )
 
@@ -26,7 +26,7 @@ filegroup(
         "emcc_link.sh",
         "link_wrapper.py",
         ":common-script-includes",
-        "@emscripten//:all",
+        "@emsdk//:binaries",
         "@nodejs//:node_files",
     ],
 )
@@ -36,7 +36,7 @@ filegroup(
     srcs = [
         ":compile-emscripten",
         ":link-emscripten",
-        "@emscripten//:all",
+        "@emsdk//:binaries",
         "@nodejs//:node_files",
     ],
 )
@@ -49,7 +49,8 @@ cc_library(name = "malloc")
 emscripten_cc_toolchain_config_rule(
     name = "wasm",
     cpu = "wasm",
-    emscripten_version = "emscripten",
+    em_config = "emscripten_config",
+    emscripten_binaries = "@emsdk//:binaries",
 )
 
 cc_toolchain(

bazel/emscripten_toolchain/crosstool.bzl

@@ -69,9 +69,10 @@ def _impl(ctx):
     abi_libc_version = "default"
 
     cc_target_os = "emscripten"
-    emscripten_version = ctx.attr.emscripten_version
 
-    builtin_sysroot = "external/emscripten/emscripten/cache/sysroot"
+    emscripten_dir = ctx.attr.emscripten_binaries.label.workspace_root
+
+    builtin_sysroot = emscripten_dir + "/emscripten/cache/sysroot"
 
     ################################################################
     # Tools
@@ -909,7 +910,7 @@ def _impl(ctx):
                 "-iwithsysroot" + "/include/c++/v1",
                 "-iwithsysroot" + "/include/compat",
                 "-iwithsysroot" + "/include",
-                "-isystem", "external/emscripten/lib/clang/13.0.0/include",
+                "-isystem", emscripten_dir + "/lib/clang/13.0.0/include",
             ],
         ),
         # Inputs and outputs
@@ -1016,6 +1017,22 @@ def _impl(ctx):
     ]
 
     crosstool_default_env_sets = [
+        # Globals
+        env_set(
+            actions = all_compile_actions +
+                      all_link_actions +
+                      [ACTION_NAMES.cpp_link_static_library],
+            env_entries = [
+                env_entry(
+                    key = "EM_BIN_PATH",
+                    value = emscripten_dir,
+                ),
+                env_entry(
+                    key = "EM_CONFIG_PATH",
+                    value = ctx.file.em_config.path,
+                ),
+            ],
+        ),
         # Use llvm backend.  Off by default, enabled via --features=llvm_backend
         env_set(
             actions = all_compile_actions +
@@ -1052,49 +1069,42 @@ def _impl(ctx):
     features.append(crosstool_default_flags_feature)
 
     cxx_builtin_include_directories = [
-        "external/emscripten/emscripten/cache/sysroot/include/c++/v1",
-        "external/emscripten/emscripten/cache/sysroot/include/compat",
-        "external/emscripten/emscripten/cache/sysroot/include",
-        "external/emscripten/lib/clang/13.0.0/include",
+        emscripten_dir + "/emscripten/cache/sysroot/include/c++/v1",
+        emscripten_dir + "/emscripten/cache/sysroot/include/compat",
+        emscripten_dir + "/emscripten/cache/sysroot/include",
+        emscripten_dir + "/lib/clang/13.0.0/include",
     ]
 
     artifact_name_patterns = []
 
     make_variables = []
 
-    out = ctx.actions.declare_file(ctx.label.name)
-    ctx.actions.write(out, "Fake executable")
-    return [
-        cc_common.create_cc_toolchain_config_info(
-            ctx = ctx,
-            features = features,
-            action_configs = action_configs,
-            artifact_name_patterns = artifact_name_patterns,
-            cxx_builtin_include_directories = cxx_builtin_include_directories,
-            toolchain_identifier = toolchain_identifier,
-            host_system_name = host_system_name,
-            target_system_name = target_system_name,
-            target_cpu = target_cpu,
-            target_libc = target_libc,
-            compiler = compiler,
-            abi_version = abi_version,
-            abi_libc_version = abi_libc_version,
-            tool_paths = tool_paths,
-            make_variables = make_variables,
-            builtin_sysroot = builtin_sysroot,
-            cc_target_os = cc_target_os,
-        ),
-        DefaultInfo(
-            executable = out,
-        ),
-    ]
+    return cc_common.create_cc_toolchain_config_info(
+        ctx = ctx,
+        features = features,
+        action_configs = action_configs,
+        artifact_name_patterns = artifact_name_patterns,
+        cxx_builtin_include_directories = cxx_builtin_include_directories,
+        toolchain_identifier = toolchain_identifier,
+        host_system_name = host_system_name,
+        target_system_name = target_system_name,
+        target_cpu = target_cpu,
+        target_libc = target_libc,
+        compiler = compiler,
+        abi_version = abi_version,
+        abi_libc_version = abi_libc_version,
+        tool_paths = tool_paths,
+        make_variables = make_variables,
+        builtin_sysroot = builtin_sysroot,
+        cc_target_os = cc_target_os,
+    )
 
 emscripten_cc_toolchain_config_rule = rule(
     implementation = _impl,
     attrs = {
         "cpu": attr.string(mandatory = True, values = ["asmjs", "wasm"]),
-        "emscripten_version": attr.string(mandatory = True),
+        "em_config": attr.label(mandatory = True, allow_single_file=True),
+        "emscripten_binaries": attr.label(mandatory = True),
     },
     provides = [CcToolchainConfigInfo],
-    executable = True,
 )

bazel/emscripten_toolchain/emar.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-source emscripten_toolchain/env.sh
+source external/emsdk/emscripten_toolchain/env.sh
 
 exec python3 $EMSCRIPTEN/emar.py "$@"

bazel/emscripten_toolchain/emcc.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-source emscripten_toolchain/env.sh
+source external/emsdk/emscripten_toolchain/env.sh
 
-exec python3 external/emscripten/emscripten/emcc.py "$@"
+exec python3 $EMSCRIPTEN/emcc.py "$@"

bazel/emscripten_toolchain/emcc_link.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-source emscripten_toolchain/env.sh
+source external/emsdk/emscripten_toolchain/env.sh
 
-exec python3 emscripten_toolchain/link_wrapper.py "$@"
+exec python3 external/emsdk/emscripten_toolchain/link_wrapper.py "$@"