[vm,dyn_modules] Hook interpreter to the standalone VM

When dynamic modules are enabled, standalone VM can now run bytecode
binaries directly. Also, if --interpreter flag is specified, Dart
source is compiled to bytecode and interpreter is used to run it.

This will allow us to test VM service capabilities including
debugging and hot reload against the interpreter.

TEST=manual

Change-Id: Ibb5a67f4844485c4ed90b8a7568dc42fa552fcac
Cq-Include-Trybots: luci.dart.try:vm-aot-dyn-linux-debug-x64-try,vm-aot-dyn-linux-product-x64-try,vm-dyn-linux-debug-x64-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/436421
Reviewed-by: Ryan Macnak <[email protected]>
Reviewed-by: Slava Egorov <[email protected]>
Commit-Queue: Alexander Markov <[email protected]>

Author: alexmarkov

pkg/dart2bytecode/bin/kernel_service.dart

@@ -0,0 +1,41 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:typed_data' show Uint8List;
+
+import 'package:dart2bytecode/bytecode_generator.dart' show generateBytecode;
+import 'package:dart2bytecode/options.dart' show BytecodeOptions;
+import 'package:kernel/ast.dart' show Component, Library;
+import 'package:kernel/binary/ast_to_binary.dart' show BytesSink;
+import 'package:kernel/class_hierarchy.dart' show ClassHierarchy;
+import 'package:kernel/core_types.dart' show CoreTypes;
+import 'package:kernel/target/targets.dart' show Target;
+
+import '../../vm/bin/kernel_service.dart' as kernel_service;
+
+Uint8List _generateBytecode(
+  Component component,
+  List<Library> libraries,
+  CoreTypes coreTypes,
+  ClassHierarchy hierarchy,
+  Target target,
+  bool enableAsserts,
+) {
+  final byteSink = new BytesSink();
+  generateBytecode(component, byteSink,
+      libraries: libraries,
+      coreTypes: coreTypes,
+      hierarchy: hierarchy,
+      target: target,
+      options: BytecodeOptions(
+          enableAsserts: enableAsserts, emitSourcePositions: true));
+  return byteSink.builder.takeBytes();
+}
+
+// Wire up bytecode generator to the kernel service to avoid
+// circular dependency between package:vm and package:dart2bytecode.
+main([args]) {
+  kernel_service.bytecodeGenerator = _generateBytecode;
+  return kernel_service.main(args);
+}

pkg/vm/bin/kernel_service.dart

@@ -77,6 +77,18 @@ const int kRejectTag = 7;
 
 bool allowDartInternalImport = false;
 
+// Bytecode generator, optionally injected in
+// pkg/dart2bytecode/bin/kernel_service.dart.
+Uint8List Function(
+  Component component,
+  List<Library> libraries,
+  CoreTypes coreTypes,
+  ClassHierarchy hierarchy,
+  Target target,
+  bool enableAsserts,
+)?
+bytecodeGenerator;
+
 CompilerOptions setupCompilerOptions(
   FileSystem fileSystem,
   Uri? platformKernelPath,
@@ -165,6 +177,7 @@ abstract class Compiler {
   final String invocationModes;
   final String verbosityLevel;
   final bool enableMirrors;
+  final bool generateBytecode;
 
   // Code coverage and hot reload are only supported by incremental compiler,
   // which is used if vm-service is enabled.
@@ -189,6 +202,7 @@ abstract class Compiler {
     this.invocationModes = '',
     this.verbosityLevel = Verbosity.defaultValue,
     required this.enableMirrors,
+    required this.generateBytecode,
   }) {
     Uri? packagesUri = null;
     final packageConfig = this.packageConfig ?? Platform.packageConfig;
@@ -309,6 +323,7 @@ class IncrementalCompilerWrapper extends Compiler {
     String invocationModes = '',
     String verbosityLevel = Verbosity.defaultValue,
     required bool enableMirrors,
+    required super.generateBytecode,
   }) : super(
          isolateGroupId,
          fileSystem,
@@ -333,6 +348,7 @@ class IncrementalCompilerWrapper extends Compiler {
     String? packageConfig,
     String invocationModes = '',
     required bool enableMirrors,
+    required bool generateBytecode,
   }) {
     IncrementalCompilerWrapper result = IncrementalCompilerWrapper(
       isolateGroupId,
@@ -343,6 +359,7 @@ class IncrementalCompilerWrapper extends Compiler {
       packageConfig: packageConfig,
       invocationModes: invocationModes,
       enableMirrors: enableMirrors,
+      generateBytecode: generateBytecode,
     );
     result.generator = new IncrementalCompiler.forExpressionCompilationOnly(
       component,
@@ -381,6 +398,7 @@ class IncrementalCompilerWrapper extends Compiler {
       packageConfig: packageConfig,
       invocationModes: invocationModes,
       enableMirrors: enableMirrors,
+      generateBytecode: generateBytecode,
     );
     final generator = this.generator!;
     // TODO(VM TEAM): This does not seem safe. What if cloning while having
@@ -424,6 +442,7 @@ class SingleShotCompilerWrapper extends Compiler {
     String invocationModes = '',
     String verbosityLevel = Verbosity.defaultValue,
     required bool enableMirrors,
+    required super.generateBytecode,
   }) : super(
          isolateGroupId,
          fileSystem,
@@ -483,6 +502,7 @@ Future<Compiler> lookupOrBuildNewIncrementalCompiler(
   String invocationModes = '',
   String verbosityLevel = Verbosity.defaultValue,
   required bool enableMirrors,
+  required bool generateBytecode,
 }) async {
   IncrementalCompilerWrapper? compiler = lookupIncrementalCompiler(
     isolateGroupId,
@@ -520,6 +540,7 @@ Future<Compiler> lookupOrBuildNewIncrementalCompiler(
         invocationModes: invocationModes,
         verbosityLevel: verbosityLevel,
         enableMirrors: enableMirrors,
+        generateBytecode: generateBytecode,
       );
     }
     isolateCompilers[isolateGroupId] = compiler;
@@ -578,6 +599,7 @@ Future _processExpressionCompilationRequest(request) async {
   final List<String>? experimentalFlags =
       request[19] != null ? request[19].cast<String>() : null;
   final bool enableMirrors = request[20];
+  final bool generateBytecode = request[21];
 
   IncrementalCompilerWrapper? compiler = isolateCompilers[isolateGroupId];
 
@@ -676,6 +698,7 @@ Future _processExpressionCompilationRequest(request) async {
           experimentalFlags: experimentalFlags,
           packageConfig: packageConfigFile,
           enableMirrors: enableMirrors,
+          generateBytecode: generateBytecode,
         );
         isolateCompilers[isolateGroupId] = compiler;
         await compiler.compile(
@@ -737,7 +760,20 @@ Future _processExpressionCompilationRequest(request) async {
       result = new CompilationResult.errors(compiler.errorsPlain);
     } else {
       Component component = createExpressionEvaluationComponent(procedure);
-      result = new CompilationResult.ok(serializeComponent(component));
+      Uint8List bytes;
+      if (compiler.generateBytecode) {
+        bytes = bytecodeGenerator!.call(
+          component,
+          component.libraries,
+          compiler.generator!.lastKnownGoodResult!.coreTypes,
+          compiler.generator!.lastKnownGoodResult!.classHierarchy,
+          compiler.options.target!,
+          compiler.enableAsserts,
+        );
+      } else {
+        bytes = serializeComponent(component);
+      }
+      result = new CompilationResult.ok(bytes);
     }
   } catch (error, stack) {
     result = new CompilationResult.crash(error, stack);
@@ -867,6 +903,7 @@ Future _processLoadRequest(request) async {
   final String? multirootScheme = request[13];
   final String verbosityLevel = request[14];
   final bool enableMirrors = request[15];
+  final bool generateBytecode = request[16];
   Uri platformKernelPath;
   List<int>? platformKernel = null;
   if (request[3] is String) {
@@ -952,6 +989,7 @@ Future _processLoadRequest(request) async {
       invocationModes: invocationModes,
       verbosityLevel: verbosityLevel,
       enableMirrors: enableMirrors,
+      generateBytecode: generateBytecode,
     );
     fileSystem = compiler.fileSystem;
   } else {
@@ -973,6 +1011,7 @@ Future _processLoadRequest(request) async {
       invocationModes: invocationModes,
       verbosityLevel: verbosityLevel,
       enableMirrors: enableMirrors,
+      generateBytecode: generateBytecode,
     );
   }
 
@@ -1028,13 +1067,30 @@ Future _processLoadRequest(request) async {
       // these sources built-in. Everything loaded as a summary in
       // [kernelForProgram] is marked `external`, so we can use that bit to
       // decide what to exclude.
-      result = new CompilationResult.ok(
-        serializeComponent(
+      Uint8List bytes;
+      if (compiler.generateBytecode) {
+        final generator = bytecodeGenerator;
+        if (generator == null) {
+          throw 'Cannot generate bytecode as dynamic modules are disabled.';
+        }
+        bytes = generator(
+          compilerResult.component!,
+          compilerResult.component!.libraries
+              .where((lib) => !loadedLibraries.contains(lib))
+              .toList(),
+          compilerResult.coreTypes!,
+          compilerResult.classHierarchy!,
+          compiler.options.target!,
+          compiler.enableAsserts,
+        );
+      } else {
+        bytes = serializeComponent(
           compilerResult.component!,
           filter: (lib) => !loadedLibraries.contains(lib),
           nativeAssetsComponent: nativeAssetsComponent,
-        ),
-      );
+        );
+      }
+      result = new CompilationResult.ok(bytes);
     }
   } catch (error, stack) {
     result = new CompilationResult.crash(error, stack);
@@ -1232,7 +1288,7 @@ Future trainInternal(String scriptUri, String? platformKernelPath) async {
     null /* multirootScheme */,
     'all' /* CFE logging mode */,
     true /* enableMirrors */,
-    null /* native assets yaml */,
+    false /* generateBytecode */,
   ];
   await _processLoadRequest(request);
 }

pkg/vm/test/kernel_service_test.dart

@@ -139,15 +139,16 @@ Future<kernel_service.Status> singleShotCompile(
     /* [4] = bool = incremental = */ false,
     /* [5] = bool = for_snapshot = */ false,
     /* [6] = bool = embed_sources = */ true,
-    /* [8] = int = isolateGroupId = */ 42,
-    /* [9] = List = sourceFiles = */ sourceFiles,
-    /* [10] = bool = enableAsserts = */ true,
-    /* [11] = List<String>? = experimentalFlags = */ [],
-    /* [12] = String? = packageConfig = */ packageConfig,
-    /* [13] = String? = multirootFilepaths = */ null,
-    /* [14] = String? = multirootScheme = */ null,
-    /* [16] = String = verbosityLevel = */ Verbosity.all.name,
-    /* [17] = bool = enableMirrors = */ false,
+    /* [7] = int = isolateGroupId = */ 42,
+    /* [8] = List = sourceFiles = */ sourceFiles,
+    /* [9] = bool = enableAsserts = */ true,
+    /* [10] = List<String>? = experimentalFlags = */ [],
+    /* [11] = String? = packageConfig = */ packageConfig,
+    /* [12] = String? = multirootFilepaths = */ null,
+    /* [13] = String? = multirootScheme = */ null,
+    /* [14] = String = verbosityLevel = */ Verbosity.all.name,
+    /* [15] = bool = enableMirrors = */ false,
+    /* [16] = bool = generateBytecode = */ false,
   ]);
 
   // Wait for kernel-service response.

runtime/bin/dart_api_win.c

@@ -111,6 +111,7 @@ typedef void (*Dart_ExitIsolateType)();
 typedef Dart_Handle (
     *Dart_CreateSnapshotType)(uint8_t**, intptr_t*, uint8_t**, intptr_t*, bool);
 typedef bool (*Dart_IsKernelType)(const uint8_t*, intptr_t);
+typedef bool (*Dart_IsBytecodeType)(const uint8_t*, intptr_t);
 typedef char* (*Dart_IsolateMakeRunnableType)(Dart_Isolate);
 typedef void (*Dart_SetMessageNotifyCallbackType)(Dart_MessageNotifyCallback);
 typedef Dart_MessageNotifyCallback (*Dart_GetMessageNotifyCallbackType)();
@@ -347,6 +348,8 @@ typedef Dart_Handle (*Dart_DeferredLoadCompleteErrorType)(intptr_t,
                                                           const char*,
                                                           bool);
 typedef Dart_Handle (*Dart_LoadScriptFromKernelType)(const uint8_t*, intptr_t);
+typedef Dart_Handle (*Dart_LoadScriptFromBytecodeType)(const uint8_t*,
+                                                       intptr_t);
 typedef Dart_Handle (*Dart_RootLibraryType)();
 typedef Dart_Handle (*Dart_SetRootLibraryType)(Dart_Handle);
 typedef Dart_Handle (*Dart_GetTypeType)(Dart_Handle,
@@ -528,6 +531,7 @@ static Dart_AddSymbolsType Dart_AddSymbolsFn = NULL;
 static Dart_ExitIsolateType Dart_ExitIsolateFn = NULL;
 static Dart_CreateSnapshotType Dart_CreateSnapshotFn = NULL;
 static Dart_IsKernelType Dart_IsKernelFn = NULL;
+static Dart_IsBytecodeType Dart_IsBytecodeFn = NULL;
 static Dart_IsolateMakeRunnableType Dart_IsolateMakeRunnableFn = NULL;
 static Dart_SetMessageNotifyCallbackType Dart_SetMessageNotifyCallbackFn = NULL;
 static Dart_GetMessageNotifyCallbackType Dart_GetMessageNotifyCallbackFn = NULL;
@@ -692,6 +696,7 @@ static Dart_DeferredLoadCompleteType Dart_DeferredLoadCompleteFn = NULL;
 static Dart_DeferredLoadCompleteErrorType Dart_DeferredLoadCompleteErrorFn =
     NULL;
 static Dart_LoadScriptFromKernelType Dart_LoadScriptFromKernelFn = NULL;
+static Dart_LoadScriptFromBytecodeType Dart_LoadScriptFromBytecodeFn = NULL;
 static Dart_RootLibraryType Dart_RootLibraryFn = NULL;
 static Dart_SetRootLibraryType Dart_SetRootLibraryFn = NULL;
 static Dart_GetTypeType Dart_GetTypeFn = NULL;
@@ -898,6 +903,8 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) {
         (Dart_CreateSnapshotType)GetProcAddress(process, "Dart_CreateSnapshot");
     Dart_IsKernelFn =
         (Dart_IsKernelType)GetProcAddress(process, "Dart_IsKernel");
+    Dart_IsBytecodeFn =
+        (Dart_IsBytecodeType)GetProcAddress(process, "Dart_IsBytecode");
     Dart_IsolateMakeRunnableFn = (Dart_IsolateMakeRunnableType)GetProcAddress(
         process, "Dart_IsolateMakeRunnable");
     Dart_SetMessageNotifyCallbackFn =
@@ -1225,6 +1232,9 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) {
             process, "Dart_DeferredLoadCompleteError");
     Dart_LoadScriptFromKernelFn = (Dart_LoadScriptFromKernelType)GetProcAddress(
         process, "Dart_LoadScriptFromKernel");
+    Dart_LoadScriptFromBytecodeFn =
+        (Dart_LoadScriptFromBytecodeType)GetProcAddress(
+            process, "Dart_LoadScriptFromBytecode");
     Dart_RootLibraryFn =
         (Dart_RootLibraryType)GetProcAddress(process, "Dart_RootLibrary");
     Dart_SetRootLibraryFn =
@@ -1648,6 +1658,10 @@ bool Dart_IsKernel(const uint8_t* buffer, intptr_t buffer_size) {
   return Dart_IsKernelFn(buffer, buffer_size);
 }
 
+bool Dart_IsBytecode(const uint8_t* buffer, intptr_t buffer_size) {
+  return Dart_IsBytecodeFn(buffer, buffer_size);
+}
+
 char* Dart_IsolateMakeRunnable(Dart_Isolate isolate) {
   return Dart_IsolateMakeRunnableFn(isolate);
 }
@@ -2369,6 +2383,11 @@ Dart_Handle Dart_LoadScriptFromKernel(const uint8_t* kernel_buffer,
   return Dart_LoadScriptFromKernelFn(kernel_buffer, kernel_size);
 }
 
+Dart_Handle Dart_LoadScriptFromBytecode(const uint8_t* kernel_buffer,
+                                        intptr_t kernel_size) {
+  return Dart_LoadScriptFromBytecodeFn(kernel_buffer, kernel_size);
+}
+
 Dart_Handle Dart_RootLibrary() {
   return Dart_RootLibraryFn();
 }

runtime/bin/dartutils.cc

@@ -44,6 +44,7 @@ MagicNumberData kernel_magic_number = {4, {0x90, 0xab, 0xcd, 0xef}};
 MagicNumberData kernel_list_magic_number = {
     7,
     {0x23, 0x40, 0x64, 0x69, 0x6c, 0x6c, 0x0a}};  // #@dill\n
+MagicNumberData bytecode_magic_number = {4, {0x33, 0x43, 0x42, 0x44}};
 MagicNumberData gzip_magic_number = {2, {0x1f, 0x8b, 0, 0}};
 
 static bool IsWindowsHost() {
@@ -398,6 +399,7 @@ DartUtils::MagicNumber DartUtils::SniffForMagicNumber(const char* filename) {
   ASSERT(aotcoff_riscv64_magic_number.length <= appjit_magic_number.length);
   ASSERT(kernel_magic_number.length <= appjit_magic_number.length);
   ASSERT(kernel_list_magic_number.length <= appjit_magic_number.length);
+  ASSERT(bytecode_magic_number.length <= appjit_magic_number.length);
   ASSERT(gzip_magic_number.length <= appjit_magic_number.length);
   if (File::GetType(nullptr, filename, true) == File::kIsFile) {
     File* file = File::Open(nullptr, filename, File::kRead);
@@ -426,8 +428,8 @@ DartUtils::MagicNumber DartUtils::SniffForMagicNumber(const uint8_t* buffer,
     return kKernelListMagicNumber;
   }
 
-  if (CheckMagicNumber(buffer, buffer_length, gzip_magic_number)) {
-    return kGzipMagicNumber;
+  if (CheckMagicNumber(buffer, buffer_length, bytecode_magic_number)) {
+    return kBytecodeMagicNumber;
   }
 
   if (CheckMagicNumber(buffer, buffer_length, gzip_magic_number)) {

runtime/bin/dartutils.h

@@ -256,6 +256,7 @@ class DartUtils {
     kAppJITMagicNumber,
     kKernelMagicNumber,
     kKernelListMagicNumber,
+    kBytecodeMagicNumber,
     kGzipMagicNumber,
     kAotELFMagicNumber,
     // Only the host-endian magic numbers are recognized, not the reverse-endian