[dart2wasm] Serialize dill before and after TFA for dynamic module enabled main modules.

Modifies the serialization strategy used for dynamic modules. Emits 2 dill files, main.dill and main.opt.dill.

main.dill serialized before TFA runs and is used by the CFE to compile the dynamic module faster (it doesn't have to recompile the main module libraries from source).

main.opt.dill is serialized at the end of the compilation of the main module and contains a copy of all the main module libraries after TFA. This dill is used by dart2wasm to produce consistent references into the main module.

Also emit new record classes into a distinct library so that all downstream code will appropriately emit the new code without special logic.

Change-Id: Idf7a87d2014dad513c753330743f753312aa472f
Tested: Just updating signature, no testing needed.
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/422342
Reviewed-by: Martin Kustermann <[email protected]>

Author: natebiggs

pkg/dart2wasm/lib/compile.dart

@@ -19,12 +19,10 @@ import 'package:front_end/src/api_unstable/vm.dart'
 import 'package:kernel/ast.dart';
 import 'package:kernel/class_hierarchy.dart';
 import 'package:kernel/core_types.dart';
-import 'package:kernel/kernel.dart'
-    show writeComponentToBinary, writeComponentToText;
+import 'package:kernel/kernel.dart' show writeComponentToText;
 import 'package:kernel/library_index.dart';
 import 'package:kernel/type_environment.dart';
 import 'package:kernel/verifier.dart';
-import 'package:path/path.dart' as path show setExtension;
 import 'package:vm/kernel_front_end.dart' show writeDepfile;
 import 'package:vm/transformations/mixin_deduplication.dart'
     as mixin_deduplication show transformLibraries;
@@ -45,7 +43,6 @@ import 'js/runtime_generator.dart' as js;
 import 'modules.dart';
 import 'record_class_generator.dart';
 import 'records.dart';
-import 'serialization.dart';
 import 'target.dart' as wasm show Mode;
 import 'target.dart' hide Mode;
 import 'translator.dart';
@@ -85,6 +82,24 @@ class CFECompileTimeErrors extends CompilationError {
   CFECompileTimeErrors();
 }
 
+const List<String> _librariesToIndex = [
+  "dart:_boxed_bool",
+  "dart:_boxed_double",
+  "dart:_boxed_int",
+  "dart:_compact_hash",
+  "dart:_internal",
+  "dart:_js_helper",
+  "dart:_js_types",
+  "dart:_list",
+  "dart:_string",
+  "dart:_wasm",
+  "dart:async",
+  "dart:collection",
+  "dart:core",
+  "dart:ffi",
+  "dart:typed_data",
+];
+
 /// Compile a Dart file into a Wasm module.
 ///
 /// Returns `null` if an error occurred during compilation. The
@@ -159,8 +174,6 @@ Future<CompilationResult> compileToModule(
     compilerOptions.compileSdk = true;
   }
 
-  final dynamicMainModuleUri = options.dynamicModuleMainUri;
-
   final dynamicModuleMainUri = await resolveUri(options.dynamicModuleMainUri);
   final dynamicInterfaceUri = await resolveUri(options.dynamicInterfaceUri);
   final isDynamicMainModule =
@@ -183,25 +196,10 @@ Future<CompilationResult> compileToModule(
 
   Component component = compilerResult!.component!;
   CoreTypes coreTypes = compilerResult.coreTypes!;
-  final classHierarchy =
+
+  ClosedWorldClassHierarchy classHierarchy =
       ClassHierarchy(component, coreTypes) as ClosedWorldClassHierarchy;
-  LibraryIndex libraryIndex = LibraryIndex(component, [
-    "dart:_boxed_bool",
-    "dart:_boxed_double",
-    "dart:_boxed_int",
-    "dart:_compact_hash",
-    "dart:_internal",
-    "dart:_js_helper",
-    "dart:_js_types",
-    "dart:_list",
-    "dart:_string",
-    "dart:_wasm",
-    "dart:async",
-    "dart:collection",
-    "dart:core",
-    "dart:ffi",
-    "dart:typed_data",
-  ]);
+  LibraryIndex libraryIndex = LibraryIndex(component, _librariesToIndex);
 
   if (options.dumpKernelAfterCfe != null) {
     writeComponentToText(component, path: options.dumpKernelAfterCfe!);
@@ -212,6 +210,43 @@ Future<CompilationResult> compileToModule(
         component, options.deleteToStringPackageUri);
   }
 
+  var jsInteropMethods = js.performJSInteropTransformations(
+      component.getDynamicModuleLibraries(coreTypes),
+      coreTypes,
+      classHierarchy);
+
+  if (isDynamicModule) {
+    // Join the dynamic module libraries with the TFAed component from the main
+    // module compilation. JS interop transformer must be run before this since
+    // some methods it uses may have been tree-shaken from the TFAed component.
+    (component, jsInteropMethods) = await generateDynamicModuleComponent(
+        component, coreTypes, dynamicModuleMainUri, jsInteropMethods);
+    coreTypes = CoreTypes(component);
+    classHierarchy =
+        ClassHierarchy(component, coreTypes) as ClosedWorldClassHierarchy;
+    libraryIndex = LibraryIndex(component, _librariesToIndex);
+  }
+
+  final librariesToTransform = isDynamicModule
+      ? component.getDynamicModuleLibraries(coreTypes)
+      : component.libraries;
+  final constantEvaluator = ConstantEvaluator(
+      options, target, component, coreTypes, classHierarchy, libraryIndex);
+  unreachable_code_elimination.transformLibraries(target, librariesToTransform,
+      constantEvaluator, options.translatorOptions.enableAsserts);
+
+  final Map<RecordShape, Class> recordClasses = generateRecordClasses(
+      component, coreTypes,
+      isDynamicMainModule: isDynamicMainModule,
+      isDynamicModule: isDynamicModule);
+  target.recordClasses = recordClasses;
+
+  if (options.dumpKernelBeforeTfa != null) {
+    writeComponentToText(component, path: options.dumpKernelBeforeTfa!);
+  }
+
+  mixin_deduplication.transformLibraries(librariesToTransform);
+
   ModuleStrategy moduleStrategy;
   if (options.translatorOptions.enableDeferredLoading) {
     moduleStrategy =
@@ -232,51 +267,19 @@ Future<CompilationResult> compileToModule(
     moduleStrategy = DefaultModuleStrategy(component);
   }
 
-  final librariesToTransform = isDynamicModule
-      ? component.getDynamicModuleLibraries(coreTypes)
-      : component.libraries;
-  ConstantEvaluator constantEvaluator = ConstantEvaluator(
-      options, target, component, coreTypes, classHierarchy, libraryIndex);
-  unreachable_code_elimination.transformLibraries(target, librariesToTransform,
-      constantEvaluator, options.translatorOptions.enableAsserts);
-
-  js.RuntimeFinalizer? jsRuntimeFinalizer = js.RuntimeFinalizer(
-      js.performJSInteropTransformations(
-          librariesToTransform, coreTypes, classHierarchy));
-
-  final Map<RecordShape, Class> recordClasses = generateRecordClasses(
-      component, coreTypes, isDynamicMainModule || isDynamicModule);
-  target.recordClasses = recordClasses;
-
-  if (options.dumpKernelBeforeTfa != null) {
-    writeComponentToText(component, path: options.dumpKernelBeforeTfa!);
-  }
-
-  mixin_deduplication.transformLibraries(librariesToTransform);
-
   moduleStrategy.prepareComponent();
 
   MainModuleMetadata mainModuleMetadata =
       MainModuleMetadata.empty(options.translatorOptions, options.environment);
 
   if (isDynamicModule) {
-    final filename = options.dynamicModuleMetadataFile ??
-        Uri.parse(
-            path.setExtension(dynamicMainModuleUri!.toFilePath(), '.dyndata'));
-    final dynamicModuleMetadataBytes =
-        await File.fromUri(filename).readAsBytes();
-    final source = DataDeserializer(dynamicModuleMetadataBytes, component);
-    mainModuleMetadata = MainModuleMetadata.deserialize(source);
+    mainModuleMetadata =
+        await deserializeMainModuleMetadata(component, options);
     mainModuleMetadata.verifyDynamicModuleOptions(options);
-    mainModuleMetadata.initializeDynamicModuleKernel(
-        component,
-        coreTypes,
-        // Create a new hierarchy with generated record classes.
-        ClassHierarchy(component, coreTypes) as ClosedWorldClassHierarchy);
   } else if (isDynamicMainModule) {
     MainModuleMetadata.verifyMainModuleOptions(options);
-    writeComponentToBinary(component, dynamicModuleMainUri.path,
-        includeSource: false);
+    await serializeMainModuleComponent(component, dynamicModuleMainUri,
+        optimized: false);
   }
 
   if (!isDynamicModule) {
@@ -327,7 +330,9 @@ Future<CompilationResult> compileToModule(
         (moduleBytes: wasmModuleSerialized, sourceMap: sourceMap);
   });
 
-  final jsRuntime = translator.isDynamicModule
+  final jsRuntimeFinalizer = js.RuntimeFinalizer(jsInteropMethods);
+
+  final jsRuntime = isDynamicModule
       ? jsRuntimeFinalizer.generateDynamicModule(
           translator.functions.translatedProcedures,
           translator.internalizedStringsForJSRuntime)
@@ -341,12 +346,9 @@ Future<CompilationResult> compileToModule(
 
   final supportJs = _generateSupportJs(options.translatorOptions);
   if (isDynamicMainModule) {
-    final filename = options.dynamicModuleMetadataFile ??
-        Uri.parse(
-            path.setExtension(dynamicMainModuleUri!.toFilePath(), '.dyndata'));
-    final serializer = DataSerializer(translator.component);
-    translator.dynamicModuleInfo!.metadata.serialize(serializer, translator);
-    await File.fromUri(filename).writeAsBytes(serializer.takeBytes());
+    await serializeMainModuleMetadata(component, translator, options);
+    await serializeMainModuleComponent(component, dynamicModuleMainUri,
+        optimized: true);
   }
 
   return CompilationSuccess(wasmModules, jsRuntime, supportJs);

pkg/dart2wasm/lib/constant_evaluator.dart

@@ -20,10 +20,10 @@ class ConstantEvaluator extends kernel.ConstantEvaluator
   final bool _hasDynamicModuleSupport;
   final bool _deferredLoadingEnabled;
 
-  final Procedure _dartInternalCheckBoundsGetter;
-  final Procedure _dartInternalMinifyGetter;
-  final Procedure _dartInternalHasDynamicModuleSupportGetter;
-  final Procedure _dartInternalDeferredLoadingEnabled;
+  final Procedure? _dartInternalCheckBoundsGetter;
+  final Procedure? _dartInternalMinifyGetter;
+  final Procedure? _dartInternalHasDynamicModuleSupportGetter;
+  final Procedure? _dartInternalDeferredLoadingEnabled;
 
   ConstantEvaluator(
       WasmCompilerOptions options,
@@ -38,15 +38,17 @@ class ConstantEvaluator extends kernel.ConstantEvaluator
         _deferredLoadingEnabled =
             options.translatorOptions.enableDeferredLoading ||
                 options.translatorOptions.enableMultiModuleStressTestMode,
-        _dartInternalCheckBoundsGetter = libraryIndex.getTopLevelProcedure(
-            "dart:_internal", "get:checkBounds"),
-        _dartInternalMinifyGetter =
-            libraryIndex.getTopLevelProcedure("dart:_internal", "get:minify"),
+        _dartInternalCheckBoundsGetter = libraryIndex.tryGetProcedure(
+            "dart:_internal", LibraryIndex.topLevel, "get:checkBounds"),
+        _dartInternalMinifyGetter = libraryIndex.tryGetProcedure(
+            "dart:_internal", LibraryIndex.topLevel, "get:minify"),
         _dartInternalHasDynamicModuleSupportGetter =
-            libraryIndex.getTopLevelProcedure(
-                "dart:_internal", "get:hasDynamicModuleSupport"),
-        _dartInternalDeferredLoadingEnabled = libraryIndex.getTopLevelProcedure(
-            "dart:_internal", "get:deferredLoadingEnabled"),
+            libraryIndex.tryGetProcedure("dart:_internal",
+                LibraryIndex.topLevel, "get:hasDynamicModuleSupport"),
+        _dartInternalDeferredLoadingEnabled = libraryIndex.tryGetProcedure(
+            "dart:_internal",
+            LibraryIndex.topLevel,
+            "get:deferredLoadingEnabled"),
         super(
           target.dartLibrarySupport,
           target.constantsBackend,

pkg/dart2wasm/lib/dispatch_table.dart

@@ -10,6 +10,7 @@ import 'package:vm/metadata/table_selector.dart';
 import 'package:wasm_builder/wasm_builder.dart' as w;
 
 import 'class_info.dart';
+import 'dynamic_module_kernel_metadata.dart';
 import 'dynamic_modules.dart';
 import 'param_info.dart';
 import 'reference_extensions.dart';
@@ -367,16 +368,25 @@ class DispatchTable {
   late final Map<TreeNode, ProcedureAttributesMetadata>
       procedureAttributeMetadata =
       translator.isDynamicModule && !isDynamicModuleTable
-          ? translator.dynamicModuleInfo!.mainModuleProcedureAttributes
+          ? (translator.component
+                      .metadata[dynamicMainModuleProcedureAttributeMetadataTag]
+                  as ProcedureAttributesMetadataRepository)
+              .mapping
           : translator.procedureAttributeMetadata;
 
   late final Translator translator;
 
   late final List<TableSelectorInfo> _selectorMetadata =
-      (translator.component.metadata["vm.table-selector.metadata"]
-              as TableSelectorMetadataRepository)
-          .mapping[translator.component]!
-          .selectors;
+      translator.isDynamicModule && !isDynamicModuleTable
+          ? (translator.component.metadata[dynamicMainModuleSelectorMetadataTag]
+                  as TableSelectorMetadataRepository)
+              .mapping[translator.component]!
+              .selectors
+          : (translator.component
+                      .metadata[TableSelectorMetadataRepository.repositoryTag]
+                  as TableSelectorMetadataRepository)
+              .mapping[translator.component]!
+              .selectors;
   late final int minClassId = isDynamicModuleTable
       ? translator.classIdNumbering.firstDynamicModuleClassId
       : 0;