Adding support for DDC's Library Bundle module system. (#4303)

This module system - in conjunction with the Frontend Server - supports web hot reload. This is part of an effort to deprecate DDC's older module systems.

Author: Markzipan

build_modules/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 5.1.5
+
+- Add support for DDC's Library Bundle module system, which is compatible with web hot reload. This is not yet enabled by default.
+
 ## 5.1.4
 
 - Fix module_builder reading DDC modules for non-primary dart libraries.

build_modules/lib/src/module_builder.dart

@@ -24,12 +24,12 @@ String moduleExtension(DartPlatform platform) => '.${platform.name}.module';
 class ModuleBuilder implements Builder {
   final DartPlatform _platform;
 
-  /// Emits DDC code with the Library Bundle module system, which supports hot
-  /// reload.
-  ///
   /// If set, this builder will consume raw meta modules (instead of clean).
-  /// Clean meta modules are only used for DDC's AMD module system due its
-  /// requirement that self-referential libraries be bundled.
+  ///
+  /// Clean meta modules cannot be used when compiling with the Frontend Server
+  /// due to potentially divergent bundling strategies between it and
+  /// build_runner. Additionally, bundling isn't required in DDC's Library
+  /// Bundle module system.
   final bool usesWebHotReload;
 
   ModuleBuilder(this._platform, {this.usesWebHotReload = false})

build_modules/pubspec.yaml

@@ -1,5 +1,5 @@
 name: build_modules
-version: 5.1.4
+version: 5.1.5
 description: >-
   Builders to analyze and split Dart code into individually compilable modules
   based on imports.

build_web_compilers/CHANGELOG.md

@@ -1,5 +1,7 @@
-## 4.4.6
+## 4.4.7
+- Add support for DDC's Library Bundle module system, which is compatible with web hot reload. This is not yet enabled by default.
 
+## 4.4.6
 - Add build options to customize the SDK used for compiling to js and wasm.
 
 ## 4.4.5

build_web_compilers/lib/builders.dart

@@ -64,8 +64,11 @@ Builder ddcBuilder(BuilderOptions options) {
     useIncrementalCompiler: _readUseIncrementalCompilerOption(options),
     generateFullDill: _readGenerateFullDillOption(options),
     emitDebugSymbols: _readEmitDebugSymbolsOption(options),
-    canaryFeatures: _readCanaryOption(options),
-    ddcModules: _readWebHotReloadOption(options),
+    canaryFeatures:
+        _readCanaryOption(options) || _readWebHotReloadOption(options),
+    ddcLibraryBundle:
+        _readDdcLibraryBundleOption(options) ||
+        _readWebHotReloadOption(options),
     trackUnusedInputs: _readTrackInputsCompilerOption(options),
     platform: ddcPlatform,
     sdkKernelPath: _readDdcKernelPathOption(options),
@@ -102,8 +105,10 @@ Builder sdkJsCompile(BuilderOptions options) {
     sdkKernelPath: 'lib/_internal/ddc_platform.dill',
     outputPath: 'lib/src/dev_compiler/dart_sdk.js',
     canaryFeatures:
-        _readWebHotReloadOption(options) || _readCanaryOption(options),
-    usesWebHotReload: _readWebHotReloadOption(options),
+        _readCanaryOption(options) || _readWebHotReloadOption(options),
+    ddcLibraryBundle:
+        _readDdcLibraryBundleOption(options) ||
+        _readWebHotReloadOption(options),
     usePrebuiltSdkFromPath: _readUsePrebuiltSdkFromPathOption(options),
   );
 }
@@ -244,6 +249,10 @@ bool _readWebHotReloadOption(BuilderOptions options) {
   return options.config[_webHotReloadOption] as bool? ?? false;
 }
 
+bool _readDdcLibraryBundleOption(BuilderOptions options) {
+  return options.config[_ddcLibraryBundleOption] as bool? ?? false;
+}
+
 bool _readUseUiLibrariesOption(BuilderOptions options) {
   return options.config[_useUiLibrariesOption] as bool? ?? false;
 }
@@ -279,6 +288,7 @@ const _canaryOption = 'canary';
 const _trackUnusedInputsCompilerOption = 'track-unused-inputs';
 const _environmentOption = 'environment';
 const _webHotReloadOption = 'web-hot-reload';
+const _ddcLibraryBundleOption = 'ddc-library-bundle';
 const _useUiLibrariesOption = 'use-ui-libraries';
 const _ddcKernelPathOption = 'ddc-kernel-path';
 const _librariesPathOption = 'libraries-path';
@@ -293,8 +303,10 @@ const _supportedOptions = [
   _canaryOption,
   _trackUnusedInputsCompilerOption,
   _webHotReloadOption,
+  _ddcLibraryBundleOption,
   _useUiLibrariesOption,
   _ddcKernelPathOption,
   _librariesPathOption,
   _platformSdkOption,
+  _usePrebuiltSdkFromPathOption,
 ];

build_web_compilers/lib/src/dev_compiler_bootstrap.dart

@@ -37,6 +37,7 @@ Future<void> bootstrapDdc(
   String entrypointExtension = jsEntrypointExtension,
   required bool? nativeNullAssertions,
   bool usesWebHotReload = false,
+  bool ddcLibraryBundle = false,
   bool unsafeAllowUnsupportedModules = false,
 }) async {
   // Ensures that the sdk resources are built and available.
@@ -112,7 +113,7 @@ $librariesString
   final dartEntrypointParts = _context.split(dartEntrypointId.path);
   final packageName = module.primarySource.package;
   final entrypointLibraryName =
-      usesWebHotReload
+      ddcLibraryBundle
           ? _context.joinAll([
             // Convert to a package: uri for files under lib.
             if (dartEntrypointParts.first == 'lib') 'package:$packageName',
@@ -132,7 +133,7 @@ $librariesString
   String entrypointJsContent;
   String bootstrapContent;
   String bootstrapEndContent;
-  if (usesWebHotReload) {
+  if (ddcLibraryBundle) {
     final ddcSdkUrl =
         r'packages/build_web_compilers/src/dev_compiler/dart_sdk.js';
     modulePaths['dart_sdk'] = ddcSdkUrl;
@@ -148,8 +149,8 @@ $librariesString
               : _context.joinAll(_context.split(jsId.path).skip(1));
     }
     final bootstrapEndModuleName = _context.relative(
-      bootstrapId.path,
-      from: _context.dirname(bootstrapEndId.path),
+      bootstrapEndId.path,
+      from: _context.dirname(bootstrapId.path),
     );
     bootstrapContent = generateDDCLibraryBundleMainModule(
       entrypoint: entrypointLibraryName,

build_web_compilers/lib/src/dev_compiler_builder.dart

@@ -42,8 +42,8 @@ class DevCompilerBuilder implements Builder {
   /// Enables canary features in DDC.
   final bool canaryFeatures;
 
-  /// Emits code with the DDC module system.
-  final bool ddcModules;
+  /// Emits code with the DDC Library Bundle module system.
+  final bool ddcLibraryBundle;
 
   final bool trackUnusedInputs;
 
@@ -74,7 +74,7 @@ class DevCompilerBuilder implements Builder {
     this.generateFullDill = false,
     this.emitDebugSymbols = false,
     this.canaryFeatures = false,
-    this.ddcModules = false,
+    this.ddcLibraryBundle = false,
     this.trackUnusedInputs = false,
     required this.platform,
     String? sdkKernelPath,
@@ -125,16 +125,16 @@ class DevCompilerBuilder implements Builder {
       await _createDevCompilerModule(
         module,
         buildStep,
-        useIncrementalCompiler,
-        generateFullDill,
-        emitDebugSymbols,
-        canaryFeatures,
-        ddcModules,
-        trackUnusedInputs,
-        platformSdk,
-        sdkKernelPath,
-        librariesPath,
         environment,
+        useIncrementalCompiler: useIncrementalCompiler,
+        generateFullDill: generateFullDill,
+        emitDebugSymbols: emitDebugSymbols,
+        canaryFeatures: canaryFeatures,
+        ddcLibraryBundle: ddcLibraryBundle,
+        trackUnusedInputs: trackUnusedInputs,
+        dartSdk: platformSdk,
+        sdkKernelPath: sdkKernelPath,
+        librariesPath: librariesPath,
       );
     } on DartDevcCompilationException catch (e) {
       await handleError(e);
@@ -148,16 +148,16 @@ class DevCompilerBuilder implements Builder {
 Future<void> _createDevCompilerModule(
   Module module,
   BuildStep buildStep,
-  bool useIncrementalCompiler,
-  bool generateFullDill,
-  bool emitDebugSymbols,
-  bool canaryFeatures,
-  bool ddcModules,
-  bool trackUnusedInputs,
-  String dartSdk,
-  String sdkKernelPath,
-  String librariesPath,
   Map<String, String> environment, {
+  required bool useIncrementalCompiler,
+  required bool generateFullDill,
+  required bool emitDebugSymbols,
+  required bool canaryFeatures,
+  required bool ddcLibraryBundle,
+  required bool trackUnusedInputs,
+  required String dartSdk,
+  required String sdkKernelPath,
+  required String librariesPath,
   bool debugMode = true,
 }) async {
   final transitiveDeps = await buildStep.trackStage(
@@ -202,11 +202,11 @@ Future<void> _createDevCompilerModule(
       WorkRequest()
         ..arguments.addAll([
           '--dart-sdk-summary=$sdkSummary',
-          '--modules=${ddcModules ? 'ddc' : 'amd'}',
+          '--modules=${ddcLibraryBundle ? 'ddc' : 'amd'}',
           '--no-summarize',
           if (generateFullDill) '--experimental-output-compiled-kernel',
           if (emitDebugSymbols) '--emit-debug-symbols',
-          if (canaryFeatures) '--canary',
+          if (canaryFeatures || ddcLibraryBundle) '--canary',
           '-o',
           jsOutputFile.path,
           debugMode ? '--source-map' : '--no-source-map',

build_web_compilers/lib/src/sdk_js_compile_builder.dart

@@ -42,9 +42,8 @@ class SdkJsCompileBuilder implements Builder {
   /// Enables canary features in DDC.
   final bool canaryFeatures;
 
-  /// Emits DDC code with the Library Bundle module system, which supports hot
-  /// reload.
-  final bool usesWebHotReload;
+  /// Emits DDC code using its Library Bundle module system.
+  final bool ddcLibraryBundle;
 
   /// An optional directory path that contains prebuilt sdk files.
   ///
@@ -58,7 +57,7 @@ class SdkJsCompileBuilder implements Builder {
     String? librariesPath,
     String? platformSdk,
     required this.canaryFeatures,
-    required this.usesWebHotReload,
+    required this.ddcLibraryBundle,
     this.usePrebuiltSdkFromPath,
   }) : platformSdk = platformSdk ?? sdkDir,
        librariesPath =
@@ -82,12 +81,12 @@ class SdkJsCompileBuilder implements Builder {
     } else {
       await _createDevCompilerModule(
         buildStep,
-        platformSdk,
-        sdkKernelPath,
-        librariesPath,
         jsOutputId,
-        canaryFeatures,
-        usesWebHotReload,
+        dartSdk: platformSdk,
+        sdkKernelPath: sdkKernelPath,
+        librariesPath: librariesPath,
+        canaryFeatures: canaryFeatures,
+        ddcLibraryBundle: ddcLibraryBundle,
       );
     }
   }
@@ -96,13 +95,13 @@ class SdkJsCompileBuilder implements Builder {
 /// Compile the sdk module with the dev compiler.
 Future<void> _createDevCompilerModule(
   BuildStep buildStep,
-  String dartSdk,
-  String sdkKernelPath,
-  String librariesPath,
-  AssetId jsOutputId,
-  bool canaryFeatures,
-  bool usesWebHotReload,
-) async {
+  AssetId jsOutputId, {
+  required String dartSdk,
+  required String sdkKernelPath,
+  required String librariesPath,
+  required bool canaryFeatures,
+  required bool ddcLibraryBundle,
+}) async {
   final scratchSpace = await buildStep.fetchResource(scratchSpaceResource);
   final jsOutputFile = scratchSpace.fileFor(jsOutputId);
 
@@ -130,8 +129,8 @@ Future<void> _createDevCompilerModule(
     result = await Process.run(dartPath, [
       snapshotPath,
       '--multi-root-scheme=org-dartlang-sdk',
-      '--modules=${usesWebHotReload ? 'ddc' : 'amd'}',
-      if (canaryFeatures || usesWebHotReload) '--canary',
+      '--modules=${ddcLibraryBundle ? 'ddc' : 'amd'}',
+      if (canaryFeatures || ddcLibraryBundle) '--canary',
       '--module-name=dart_sdk',
       '-o',
       jsOutputFile.path,

build_web_compilers/lib/src/web_entrypoint_builder.dart

@@ -126,9 +126,13 @@ final class EntrypointBuilderOptions {
 
   /// Whether or not to emit DDC entrypoints that support web hot reload.
   ///
-  /// Web hot reload is only supported for DDC's Library Bundle module system.
+  /// Only supported for DDC's Library Bundle module system.
   final bool usesWebHotReload;
 
+  /// Whether or not to emit DDC entrypoints that target the DDC Library Bundle
+  /// module system.
+  final bool ddcLibraryBundle;
+
   /// The absolute path to the libraries file for the current platform.
   ///
   /// If not provided, defaults to "lib/libraries.json" in the sdk directory.
@@ -145,6 +149,7 @@ final class EntrypointBuilderOptions {
     this.nativeNullAssertions,
     this.loaderExtension,
     this.usesWebHotReload = false,
+    this.ddcLibraryBundle = false,
     this.librariesPath,
     this.unsafeAllowUnsupportedModules = false,
   });
@@ -159,9 +164,11 @@ final class EntrypointBuilderOptions {
     const nativeNullAssertionsOption = 'native_null_assertions';
     const loaderOption = 'loader';
     const webHotReloadOption = 'web-hot-reload';
+    const ddcLibraryBundleOption = 'ddc-library-bundle';
     const librariesPathOption = 'libraries-path';
     const unsafeAllowUnsupportedModulesOption =
         'unsafe-allow-unsupported-modules';
+
     String? defaultLoaderOption;
 
     const supportedOptions = [
@@ -172,6 +179,7 @@ final class EntrypointBuilderOptions {
       dart2wasmArgsOption,
       loaderOption,
       webHotReloadOption,
+      ddcLibraryBundleOption,
       librariesPathOption,
       unsafeAllowUnsupportedModulesOption,
       'use-ui-libraries',
@@ -181,9 +189,12 @@ final class EntrypointBuilderOptions {
     final nativeNullAssertions =
         options.config[nativeNullAssertionsOption] as bool?;
     final usesWebHotReload = options.config[webHotReloadOption] as bool?;
+    final usesDdcLibraryBundle =
+        usesWebHotReload ?? options.config[ddcLibraryBundleOption] as bool?;
     final librariesPath = options.config[librariesPathOption] as String?;
     final unsafeAllowUnsupportedModules =
         options.config[unsafeAllowUnsupportedModulesOption] as bool?;
+
     final compilers = <EnabledEntrypointCompiler>[];
 
     validateOptions(
@@ -271,6 +282,7 @@ final class EntrypointBuilderOptions {
               ? config[loaderOption] as String?
               : defaultLoaderOption,
       usesWebHotReload: usesWebHotReload ?? false,
+      ddcLibraryBundle: usesDdcLibraryBundle ?? false,
       librariesPath: librariesPath,
       unsafeAllowUnsupportedModules: unsafeAllowUnsupportedModules ?? false,
     );
@@ -354,14 +366,17 @@ class WebEntrypointBuilder implements Builder {
             Future(() async {
               try {
                 final usesWebHotReload = options.usesWebHotReload;
+                final usesDdcLibraryBundle =
+                    options.ddcLibraryBundle || usesWebHotReload;
                 await bootstrapDdc(
                   buildStep,
                   nativeNullAssertions: options.nativeNullAssertions,
                   requiredAssets:
-                      usesWebHotReload
+                      usesDdcLibraryBundle
                           ? _ddcLibraryBundleSdkResources
                           : _ddcSdkResources,
                   usesWebHotReload: usesWebHotReload,
+                  ddcLibraryBundle: usesDdcLibraryBundle,
                   unsafeAllowUnsupportedModules:
                       options.unsafeAllowUnsupportedModules,
                 );

build_web_compilers/lib/src/web_entrypoint_marker_builder.dart

@@ -11,7 +11,7 @@ import 'package:glob/glob.dart';
 /// A builder that gathers information about a web target's 'main' entrypoint.
 class WebEntrypointMarkerBuilder implements Builder {
   /// Records state (such as the web entrypoint) required when compiling DDC
-  /// with the Library Bundle module system.
+  /// with the Frontend Server, which supports hot reload.
   ///
   /// A no-op if [usesWebHotReload] is not set.
   final bool usesWebHotReload;