build: generate linked ESM bundles of framework packages

As of APF v13, Angular packages come with partial compilation output.
This means that for tests, or running the dev-app we want to run the
linker to avoid JIT compilation of the FW code.

This commit introduces a macro for generating linked ESM bundles of
framework packages and their entry-points. These bundles can then
be passed later to tests or the dev-app, so that it would not need
to run the linker at all. This helps speeding up the development
turnaround as the linker would not need to re-run *always*..

Author: devversion

package.json

@@ -146,6 +146,7 @@
     "@rollup/plugin-commonjs": "^20.0.0",
     "@rollup/plugin-node-resolve": "^13.0.5",
     "@schematics/angular": "13.0.0-next.7",
+    "@types/babel__core": "^7.1.16",
     "@types/browser-sync": "^2.26.1",
     "@types/fs-extra": "^9.0.5",
     "@types/glob": "^7.1.3",

tools/angular/BUILD.bazel

@@ -0,0 +1,24 @@
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+load("//tools/esbuild:index.bzl", "esbuild_config")
+load(":index.bzl", "create_angular_bundle_targets")
+
+package(default_visibility = ["//visibility:public"])
+
+js_library(
+    name = "create_linker_esbuild_plugin",
+    srcs = ["create_linker_esbuild_plugin.mjs"],
+    deps = [
+        "@npm//@angular/compiler-cli",
+        "@npm//@babel/core",
+    ],
+)
+
+esbuild_config(
+    name = "esbuild_config",
+    config_file = "esbuild.config.mjs",
+    deps = [
+        ":create_linker_esbuild_plugin",
+    ],
+)
+
+create_angular_bundle_targets()

tools/angular/create_linker_esbuild_plugin.mjs

@@ -0,0 +1,81 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import fs from 'fs';
+
+/** Naively checks whether this node path resolves to an Angular declare invocation. */
+function isNgDeclareCallExpression(nodePath) {
+  if (!nodePath.node.name.startsWith('ɵɵngDeclare')) {
+    return false;
+  }
+
+  // Expect the `ngDeclare` identifier to be used as part of a property access that
+  // is invoked within a call expression. e.g. `i0.ɵɵngDeclare<>`.
+  return nodePath.parentPath?.type === 'MemberExpression' &&
+    nodePath.parentPath.parentPath?.type === 'CallExpression';
+}
+
+/** Asserts that the given AST does not contain any Angular partial declaration. */
+async function assertNoPartialDeclaration(filePath, ast, traverseFn) {
+  // Naively check if there are any Angular declarations left that haven't been linked.
+  traverseFn(ast, {
+    Identifier: (astPath) => {
+      if (isNgDeclareCallExpression(astPath)) {
+        throw astPath.buildCodeFrameError(
+          `Found Angular declaration that has not been linked. ${filePath}`, Error);
+      }
+    }
+  });
+}
+
+/**
+ * Creates an ESBuild plugin for running the Angular linker resolved sources.
+ *
+ * @param filter Mandatory file path filter for the ESBuild plugin to apply to. Read
+ *   more here: https://esbuild.github.io/plugins/#filters.
+ * @param ensureNoPartialDeclaration Whether an additional check ensuring there are
+ *   no partial declarations should run.
+ */
+export async function createLinkerEsbuildPlugin(filter, ensureNoPartialDeclaration) {
+  // Note: We load all dependencies asynchronously so that these large dependencies
+  // do not slow-down bundling when the linker plugin is not actually created.
+  const {NodeJSFileSystem, ConsoleLogger, LogLevel} = await import('@angular/compiler-cli');
+  const {createEs2015LinkerPlugin} = await import('@angular/compiler-cli/linker/babel');
+  const {default: babel} = await import('@babel/core');
+
+  const linkerBabelPlugin = createEs2015LinkerPlugin({
+    fileSystem: new NodeJSFileSystem(),
+    logger: new ConsoleLogger(LogLevel.warn),
+    // We enable JIT mode as unit tests also will rely on the linked ESM files.
+    linkerJitMode: true,
+  });
+
+  return {
+    name: 'ng-linker-esbuild',
+    setup: (build) => {
+      build.onLoad({filter}, async (args) => {
+        const filePath = args.path;
+        const content = await fs.promises.readFile(filePath, 'utf8');
+        const {ast, code} = await babel.transformAsync(content, {
+          filename: filePath,
+          filenameRelative: filePath,
+          plugins: [linkerBabelPlugin],
+          sourceMaps: 'inline',
+          ast: true,
+          compact: false,
+        });
+
+        if (ensureNoPartialDeclaration) {
+          await assertNoPartialDeclaration(filePath, ast, babel.traverse);
+        }
+
+        return {contents: code};
+      });
+    },
+  };
+}

tools/angular/esbuild.config.mjs

@@ -0,0 +1,19 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {createLinkerEsbuildPlugin} from './create_linker_esbuild_plugin.mjs';
+
+export default {
+  resolveExtensions: ['.mjs', '.js'],
+  format: 'esm',
+  plugins: [
+    // Only run the linker on `fesm2020/` bundles. This should not have an effect on
+    // the bundle output, but helps speeding up ESBuild when it visits other modules.
+    await createLinkerEsbuildPlugin(/fesm2020/)
+  ]
+};

tools/angular/index.bzl

@@ -0,0 +1,78 @@
+load("//:packages.bzl", "ANGULAR_PACKAGES")
+load("//tools/esbuild:index.bzl", "esbuild")
+load("@build_bazel_rules_nodejs//internal/linker:link_node_modules.bzl", "LinkerPackageMappingInfo")
+load("@build_bazel_rules_nodejs//:providers.bzl", "ExternalNpmPackageInfo", "JSModuleInfo")
+
+"""
+  Starlark file exposing a definition for generating Angular linker-processed ESM bundles
+  for all entry-points the Angular framework packages expose.
+
+  These linker-processed ESM bundles are useful as they can be integrated into the
+  spec bundling, or dev-app to avoid unnecessary re-linking of framework entry-points
+  every time the bundler executes. This helps with the overall development turnaround and
+  is more idiomatic as it allows caching of the Angular framework packages.
+"""
+
+def _linker_mapping_impl(ctx):
+    return [
+        # Pass through the `ExternalNpmPackageInfo` which is needed for the linker
+        # resolving dependencies which might be external. e.g. `rxjs` for `@angular/core`.
+        ctx.attr.package[ExternalNpmPackageInfo],
+        JSModuleInfo(
+            direct_sources = depset(ctx.files.srcs),
+            sources = depset(ctx.files.srcs),
+        ),
+        LinkerPackageMappingInfo(
+            mappings = {
+                ctx.attr.module_name: "%s/%s" % (ctx.label.package, ctx.attr.subpath),
+            },
+        ),
+    ]
+
+_linker_mapping = rule(
+    implementation = _linker_mapping_impl,
+    attrs = {
+        "package": attr.label(),
+        "srcs": attr.label_list(allow_files = False),
+        "subpath": attr.string(),
+        "module_name": attr.string(),
+    },
+)
+
+def _get_target_name_base(pkg, entry_point):
+    return "%s%s" % (pkg.name, "_%s" % entry_point if entry_point else "")
+
+def _create_bundle_targets(pkg, entry_point, module_name):
+    target_name_base = _get_target_name_base(pkg, entry_point)
+    fesm_bundle_path = "fesm2020/%s.mjs" % (entry_point if entry_point else pkg.name)
+
+    esbuild(
+        name = "%s_linked_bundle" % target_name_base,
+        output = "%s/index.mjs" % target_name_base,
+        platform = pkg.platform,
+        entry_point = "@npm//:node_modules/@angular/%s/%s" % (pkg.name, fesm_bundle_path),
+        config = "//tools/angular:esbuild_config",
+        # List of dependencies which should never be bundled into these linker-processed bundles.
+        external = ["rxjs", "@angular", "domino", "xhr2"],
+    )
+
+    _linker_mapping(
+        name = "%s_linked" % target_name_base,
+        srcs = [":%s_linked_bundle" % target_name_base],
+        package = "@npm//@angular/%s" % pkg.name,
+        module_name = module_name,
+        subpath = target_name_base,
+    )
+
+def create_angular_bundle_targets():
+    for pkg in ANGULAR_PACKAGES:
+        _create_bundle_targets(pkg, None, pkg.module_name)
+
+        for entry_point in pkg.entry_points:
+            _create_bundle_targets(pkg, entry_point, "%s/%s" % (pkg.module_name, entry_point))
+
+LINKER_PROCESSED_FW_PACKAGES = [
+    "//tools/angular:%s_linked" % _get_target_name_base(pkg, entry_point)
+    for pkg in ANGULAR_PACKAGES
+    for entry_point in [None] + pkg.entry_points
+]

tools/esbuild/devmode-output.bzl

@@ -0,0 +1,43 @@
+load("@build_bazel_rules_nodejs//internal/linker:link_node_modules.bzl", "LinkerPackageMappingInfo", "module_mappings_aspect")
+load("@build_bazel_rules_nodejs//:providers.bzl", "ExternalNpmPackageInfo", "JSModuleInfo", "node_modules_aspect")
+
+def _extract_devmode_sources_impl(ctx):
+    """Private rule that extracts devmode sources for all direct dependencies and re-exposes
+      them as part of a single target. External node modules are passed-through as well."""
+
+    mappings = {}
+    js_sources = []
+
+    for dep in ctx.attr.deps:
+        if JSModuleInfo in dep:
+            js_sources.append(dep[JSModuleInfo].sources)
+        if ExternalNpmPackageInfo in dep:
+            js_sources.append(dep[ExternalNpmPackageInfo].sources)
+        if LinkerPackageMappingInfo in dep:
+            mappings.update(dep[LinkerPackageMappingInfo].mappings)
+
+    return [
+        LinkerPackageMappingInfo(mappings = mappings),
+        JSModuleInfo(
+            direct_sources = depset(transitive = js_sources),
+            sources = depset(transitive = js_sources),
+        ),
+    ]
+
+extract_devmode_sources = rule(
+    implementation = _extract_devmode_sources_impl,
+    attrs = {
+        "deps": attr.label_list(mandatory = True, aspects = [module_mappings_aspect, node_modules_aspect]),
+    },
+)
+
+def extract_devmode_output_with_mappings(name, deps, testonly):
+    """Macro that extracts devmode ESM2020 sources from the given dependencies."""
+
+    extract_devmode_sources(
+        name = "%s_sources" % name,
+        testonly = testonly,
+        deps = deps,
+    )
+
+    return ["%s_sources" % name]

tools/esbuild/esbuild-amd-config.mjs

@@ -1,5 +1,24 @@
+import url from 'url';
+import path from 'path';
+
+/** Path to the ESBuild configuration maintained by the user. */
+const userConfigExecPath = "TMPL_CONFIG_PATH"
+
+/** User ESBuild config. Empty if none is loaded. */
+let userConfig = {};
+
+if (userConfigExecPath !== '') {
+  const userConfigPath = path.join(process.cwd(), userConfigExecPath);
+  const userConfigUrl = url.pathToFileURL(userConfigPath);
+
+  // Load the user config, assuming it is set as `default` export.
+  userConfig = (await import(userConfigUrl)).default;
+}
+
 export default {
+  ...userConfig,
   globalName: "__exports",
+  format: 'iife',
   banner: {js: 'define("TMPL_MODULE_NAME", [], function() {'},
   footer: {js: 'return __exports;})'},
 };

tools/esbuild/index.bzl

@@ -1,42 +1,52 @@
-load("@npm//@bazel/esbuild:index.bzl", "esbuild_config", _esbuild = "esbuild")
+load("@npm//@bazel/esbuild:index.bzl", _esbuild = "esbuild", _esbuild_config = "esbuild_config")
 load("@npm//@angular/dev-infra-private/bazel:expand_template.bzl", "expand_template")
+load("//tools/esbuild:devmode-output.bzl", "extract_devmode_output_with_mappings")
 
-def esbuild(**kwargs):
-    _esbuild(**kwargs)
+# Re-export of the actual esbuild definitions.
+esbuild_config = _esbuild_config
+
+def esbuild(name, deps = [], mapping_targets = [], testonly = False, **kwargs):
+    # Extract all JS module sources before passing to ESBuild. The ESBuild rule requests
+    # both the devmode and prodmode unfortunately and this would slow-down the development
+    # turnaround significantly. We only request the devmode sources which are ESM as well.
+    devmode_targets = extract_devmode_output_with_mappings(name, deps, testonly)
+
+    _esbuild(
+        name = name,
+        deps = devmode_targets,
+        testonly = testonly,
+        **kwargs
+    )
 
 """Generates an AMD bundle for the specified entry-point with the given AMD module name."""
 
-def esbuild_amd(name, entry_point, module_name, testonly, deps):
+def esbuild_amd(name, entry_point, module_name, testonly = False, config = None, deps = [], **kwargs):
     expand_template(
         name = "%s_config" % name,
         testonly = testonly,
         template = "//tools/esbuild:esbuild-amd-config.mjs",
         output_name = "%s_config.mjs" % name,
         substitutions = {
             "TMPL_MODULE_NAME": module_name,
+            "TMPL_CONFIG_PATH": "$(execpath %s)" % config if config else "",
         },
+        data = [config] if config else None,
     )
 
-    esbuild_config(
+    _esbuild_config(
         name = "%s_config_lib" % name,
         testonly = testonly,
         config_file = "%s_config" % name,
+        # Adds the user configuration and its deps as dependency of the AMD ESBuild config.
+        # https://github.com/bazelbuild/rules_nodejs/blob/a892caf5a040bae5eeec174a3cf6250f02caf364/packages/esbuild/esbuild_config.bzl#L23.
+        deps = [config, "%s_deps" % config] if config else None,
     )
 
-    _esbuild(
-        name = "%s_bundle" % name,
+    esbuild(
+        name = name,
         testonly = testonly,
         deps = deps,
-        minify = True,
-        sourcemap = "inline",
-        platform = "browser",
-        target = "es2015",
         entry_point = entry_point,
         config = "%s_config_lib" % name,
-    )
-
-    native.filegroup(
-        name = name,
-        testonly = testonly,
-        srcs = ["%s_bundle" % name],
+        **kwargs
     )