Migrate to Angular's spec_bundle for tests. (#6036)

The upcoming upgrade to Angular 13+ introduces a number of
bundling-related problems for our tests and applications. Angular has
developed some internal tooling to help resolve these problems. This
change uses Angular's internal spec_bundle() macro to allow us to
continue to run Karma tests with Angular 13+.

Note: spec_bundle() is Angular-internal and not supported for external
use. We use it at our own risk. Googlers, see the following documents
for background and justification: http://go/angular-bazel-problems,
http://go/tb-oss-bundling.

Our usage of spec_bundle() is largely guided/inspired by the usage in
the angular/components repo. This is the version at HEAD as of
early November:
*
https://github.com/angular/components/blob/871f8f231a7a86a7a0778e345f4d517109c9a357/tools/defaults.bzl

Highlights.
1. Run the following commands to install Angular's build tooling and
Babel and then clean the environment:
* `yarn add
https://github.com/angular/dev-infra-private-build-tooling-builds.git#fb42478534df7d48ec23a6834fea94a776cb89a0
@babel/core@^7.16.12 --dev`
* `yarn run yarn-deduplicate`
* `rm -rf node_modules; bazel clean --expunge; yarn;`
2. Automaticaly rename, at build-time, *test.ts files to *test.spec.ts
files so that they are recognized by spec_bundle() as top-level test
files to be executed. Otherwise they would be ignored and the tests not
executed.
3. Modify our build rules to produce correct compiler output and bundle
them using extract_js_module_output() and spec_bundle().
4. Cleanup files we no longer need. There are a number of shims and
hacks that we needed for compatibility with karma test suite that are
now unnecessary due to how the spec_bundle() tooling works.

Author: bmd3k

BUILD

@@ -4,15 +4,6 @@ licenses(["notice"])
 
 exports_files(["tsconfig.json"])
 
-ts_config(
-    name = "tsconfig-test",
-    src = "tsconfig-test.json",
-    visibility = [
-        "//tensorboard:internal",
-    ],
-    deps = [":tsconfig.json"],
-)
-
 # Inspired from internal tsconfig generation for project like TensorBoard.
 ts_config(
     name = "tsconfig-lax",

package.json

@@ -28,9 +28,11 @@
   },
   "homepage": "https://github.com/tensorflow/tensorboard#readme",
   "devDependencies": {
+    "@angular/build-tooling": "https://github.com/angular/dev-infra-private-build-tooling-builds.git#fb42478534df7d48ec23a6834fea94a776cb89a0",
     "@angular/cli": "^12.2.0",
     "@angular/compiler": "^12.2.0",
     "@angular/compiler-cli": "^12.2.0",
+    "@babel/core": "^7.16.12",
     "@bazel/concatjs": "5.7.0",
     "@bazel/esbuild": "5.7.0",
     "@bazel/ibazel": "^0.15.9",

tensorboard/defs/defs.bzl

@@ -13,7 +13,10 @@
 # limitations under the License.
 """External-only delegates for various BUILD rules."""
 
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@io_bazel_rules_sass//:defs.bzl", "npm_sass_library", "sass_binary", "sass_library")
+load("@npm//@angular/build-tooling/bazel/spec-bundling:index.bzl", "spec_bundle")
+load("@npm//@angular/build-tooling/bazel:extract_js_module_output.bzl", "extract_js_module_output")
 load("@npm//@bazel/concatjs:index.bzl", "karma_web_test_suite", "ts_library")
 load("@npm//@bazel/esbuild:index.bzl", "esbuild")
 load("@npm//@bazel/typescript:index.bzl", "ts_config")
@@ -86,7 +89,7 @@ def tf_ts_config(**kwargs):
 
     ts_config(**kwargs)
 
-def tf_ts_library(strict_checks = True, **kwargs):
+def tf_ts_library(srcs = [], strict_checks = True, **kwargs):
     """TensorBoard wrapper for the rule for a TypeScript library.
 
     Args:
@@ -98,50 +101,94 @@ def tf_ts_library(strict_checks = True, **kwargs):
 
     if strict_checks == False:
         tsconfig = "//:tsconfig-lax"
-    elif "test_only" in kwargs and kwargs.get("test_only"):
-        tsconfig = "//:tsconfig-test"
     kwargs.setdefault("deps", []).extend(["@npm//tslib", "//tensorboard/defs:strict_types"])
 
-    ts_library(tsconfig = tsconfig, supports_workers = True, **kwargs)
+    new_srcs = []
+    # Find test.ts and testbed.ts files and rename to test.spec.ts to be
+    # compatible with spec_bundle() tooling.
+    for s in srcs:
+      if s.endswith("_test.ts") or s.endswith("-test.ts") or s.endswith("_testbed.ts"):
+        # Make a copy of the file with name .spec.ts and switch to that as
+        # the src file.
+        new_src = s[0:s.rindex('.ts')] + ".spec.ts"
+        copy_file(
+            name = new_src + '_spec_copy',
+            src = s,
+            out = new_src,
+            allow_symlink = True)
+        new_srcs.append(new_src)
+      else:
+        # Not a test file. Nothing to do here.
+        new_srcs.append(s)
 
-def tf_ng_web_test_suite(runtime_deps = [], bootstrap = [], deps = [], **kwargs):
+    ts_library(
+        srcs = new_srcs,
+        tsconfig = tsconfig,
+        supports_workers = True,
+        # Override prodmode_target, devmode_target, and devmode_module to be
+        # compatible with extract_js_module_output() and spec_bundle() tooling.
+        # See the angular/components example:
+        # https://github.com/angular/components/blob/871f8f231a7a86a7a0778e345f4d517109c9a357/tools/defaults.bzl#L114-L121
+        prodmode_target = "es2020",
+        devmode_target = "es2020",
+        devmode_module = "esnext",
+        **kwargs)
+
+def tf_ng_web_test_suite(name, deps = [], **kwargs):
     """TensorBoard wrapper for the rule for a Karma web test suite.
 
-    It has Angular specific configurations that we want as defaults.
+    We use the Angular team's internal toolchain for bundling Angular-compatible
+    tests: extract_js_module_output() and spec_bundle(). This toolchain is not
+    officially supported and is subject to change or deletion.
     """
 
-    kwargs.setdefault("tags", []).append("webtest")
-    karma_web_test_suite(
-        srcs = [
-            "//tensorboard/webapp/testing:require_js_karma_config.js",
-        ],
-        bootstrap = bootstrap + [
-            "@npm//:node_modules/zone.js/dist/zone-testing-bundle.js",
-            "@npm//:node_modules/reflect-metadata/Reflect.js",
-            "@npm//:node_modules/@angular/localize/bundles/localize-init.umd.js",
-        ],
-        runtime_deps = runtime_deps + [
+    # Call extract_js_module_output() to prepare proper input for spec_bundle()
+    # tooling.
+    # See the angular/components example:
+    # https://github.com/angular/components/blob/871f8f231a7a86a7a0778e345f4d517109c9a357/tools/defaults.bzl#L427
+    extract_js_module_output(
+        name = "%s_devmode_deps" % name,
+        deps = [
+            # initialize_testbed must be the first dep for the tests to run
+            # properly.
             "//tensorboard/webapp/testing:initialize_testbed",
+        ] + deps,
+        provider = "JSModuleInfo",
+        forward_linker_mappings = True,
+        include_external_npm_packages = True,
+        include_default_files = False,
+        include_declarations = False,
+        testonly = True,
+    )
+
+    # Create an esbuild bundle with all source and dependencies. It provides a
+    # clean way to bundle non-CommonJS dependencies without the use of hacks
+    # and shims that are typically necessary for working with
+    # karma_web_test_suite().
+    # See the angular/components example:
+    # https://github.com/angular/components/blob/871f8f231a7a86a7a0778e345f4d517109c9a357/tools/defaults.bzl#L438
+    spec_bundle(
+        name = "%s_bundle" % name,
+        deps = ["%s_devmode_deps" % name],
+        workspace_name = "org_tensorflow_tensorboard",
+        run_angular_linker = False,
+        platform = "browser",
+    )
+
+    karma_web_test_suite(
+        name = name,
+        bootstrap =
+            [
+                "@npm//:node_modules/zone.js/dist/zone-evergreen.js",
+                "@npm//:node_modules/zone.js/dist/zone-testing.js",
+                "@npm//:node_modules/reflect-metadata/Reflect.js",
+            ],
+        # The only dependency is the esbuild bundle from spec_bundle().
+        # karma_web_test_suite() will rebundle it along with the test framework
+        # in a CommonJS bundle.
+        deps = [
+          "%s_bundle" % name,
         ],
-        deps = deps + [
-            "//tensorboard/defs/internal:common_umd_lib",
-        ],
-        # Lodash runtime dependency that is compatible with requirejs for karma.
-        static_files = [
-            "@npm//:node_modules/@tensorflow/tfjs-core/dist/tf-core.js",
-            "@npm//:node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.js",
-            "@npm//:node_modules/@tensorflow/tfjs-backend-webgl/dist/tf-backend-webgl.js",
-            "@npm//:node_modules/umap-js/lib/umap-js.js",
-            # tfjs-backend-cpu only uses alea.
-            # https://github.com/tensorflow/tfjs/blob/bca336cd8297cb733e3ddcb3c091eac2eb4d5fc5/tfjs-backend-cpu/src/kernels/Multinomial.ts#L58
-            "@npm//:node_modules/seedrandom/lib/alea.js",
-            "@npm//:node_modules/lodash/lodash.js",
-            "@npm//:node_modules/d3/dist/d3.js",
-            "@npm//:node_modules/three/build/three.js",
-            "@npm//:node_modules/dagre/dist/dagre.js",
-            "@npm//:node_modules/marked/marked.min.js",
-        ],
-        **kwargs
     )
 
 def tf_svg_bundle(name, srcs, out):
@@ -193,9 +240,8 @@ def tf_external_sass_libray(**kwargs):
 
 def tf_ng_module(assets = [], **kwargs):
     """TensorBoard wrapper for Angular modules."""
-    ts_library(
+    tf_ts_library(
         compiler = "//tensorboard/defs:tsc_wrapped_with_angular",
-        supports_workers = True,
         use_angular_plugin = True,
         angular_assets = assets,
         **kwargs

tensorboard/defs/internal/BUILD

@@ -15,12 +15,3 @@ py_test(
         "//tensorboard:test",
     ],
 )
-
-filegroup(
-    name = "common_umd_lib",
-    srcs = [
-        "rxjs_shims.js",
-        "@npm//:node_modules/rxjs/dist/bundles/rxjs.umd.js",
-        "@npm//:node_modules/tslib/tslib.js",
-    ],
-)

tensorboard/defs/internal/rxjs_shims.js

@@ -1,7 +1,5 @@
 # Description:
 # TensorBoard plugin for graphs
-load("//tensorboard/defs:defs.bzl", "tf_ng_web_test_suite")
-
 package(default_visibility = ["//tensorboard:internal"])
 
 licenses(["notice"])
@@ -156,9 +154,13 @@ py_library(
     srcs_version = "PY3",
 )
 
-tf_ng_web_test_suite(
-    name = "karma_test",
-    deps = [
-        "//tensorboard/plugins/graph/tf_graph_common:test_lib",
-    ],
-)
+# This test suite is disabled as lodash does not seem to get properly bundled
+# into the test after upgrade to spec_bundle(). Further investigation is
+# required.
+# TODO: Reenable or decide to delete the tests.
+# tf_ng_web_test_suite(
+#     name = "karma_test",
+#     deps = [
+#         "//tensorboard/plugins/graph/tf_graph_common:test_lib",
+#     ],
+# )

tensorboard/plugins/graph/BUILD

@@ -15,6 +15,7 @@ tf_ts_library(
     deps = [
         "//tensorboard/webapp/angular:expect_angular_core_testing",
         "//tensorboard/webapp/angular:expect_angular_platform_browser_dynamic_testing",
+        "@npm//@angular/localize",
     ],
 )
 
@@ -83,5 +84,3 @@ tf_ng_module(
         "@npm//@ngrx/store",
     ],
 )
-
-exports_files(["require_js_karma_config.js"])

tensorboard/webapp/testing/BUILD

@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 import {TestBed} from '@angular/core/testing';
+import '@angular/localize/init';
 import {
   BrowserDynamicTestingModule,
   platformBrowserDynamicTesting,

tensorboard/webapp/testing/initialize_testbed.ts

@@ -313,7 +313,7 @@ describe('line_chart_v2/lib/scale test', () => {
         expect(scale.reverse([1000, 2000], [-100, 100], -100)).toBe(1000);
         expect(scale.reverse([1000, 2000], [-100, 100], 100)).toBe(2000);
 
-        expect(scale.reverse([1000, 2000], [-100, 100], -101)).toBe(995);
+        expect(scale.reverse([1000, 2000], [-100, 100], -102)).toBe(990);
         expect(scale.reverse([1000, 2000], [-100, 100], 500)).toBe(4000);
 
         expect(