[VM/dartdev] Switch dartdev to use an AOT runtime.

- split the Dart CLI tool out of the VM into it's own embedder which
  runs in AOT mode. The pure Dart VM executable is called 'dartvm' and
  has no Dart CLI functionality in it
- the Dart CLI executable parses the CLI commands and invokes the rest
  of the AOT tools in the same process, for the 'run' and 'test'
  commands it execs a process which runs 'dartvm' to run
- 'dart hello.dart' execs the 'dartvm' process and runs 'hello.dart'
- the Dart CLI is not generated for ia32 as we are not shipping a
  Dart SDK for ia32 anymore (support to execute the 'dartvm' for ia32
  architecture is retained)
- the Dart CLI tool is not built in the internal Dart SDK builds

TEST=ci

Some performance improvement numbers
'dart format pkg/dartdev' goes from 1.17 secs to 0.22 secs
'dart doc pkg/dartdev' goes from 100.2 secs to 66.6 secs
'dart fix pkg/dartdev' goes from 19.3 secs to 14.5 secs

Change-Id: I66984a26cb2ab014b34dc1873f1f3d2884e13518
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/364202
Commit-Queue: Ben Konyi <[email protected]>
Reviewed-by: Ben Konyi <[email protected]>

Author: a-siva

BUILD.gn

@@ -42,6 +42,7 @@ group("runtime") {
 
   deps = [
     "runtime/bin:dart",
+    "runtime/bin:dartvm",
     "runtime/bin:ffi_test_dynamic_library",
     "runtime/bin:ffi_test_functions",
     "runtime/bin:process_test",
@@ -60,12 +61,15 @@ group("runtime") {
     deps += [
       "runtime/bin:dartaotruntime",
       "runtime/bin:dartaotruntime_product",
+      "utils/dartdev:dartdev_aot_snapshot",
       "utils/dds:dds_aot",
+      "utils/dtd:dtd_aot",
       "utils/kernel-service:frontend_server_aot_product",
     ]
   } else {
     deps += [
       "utils/dds:dds",
+      "utils/dtd:dtd",
       "utils/kernel-service:frontend_server",
     ]
   }
@@ -420,7 +424,7 @@ if (is_fuchsia) {
   }
 
   test_binaries = [
-    "dart",
+    "dartvm",
     "dartaotruntime",
     "run_vm_tests",
   ]

CHANGELOG.md

@@ -75,6 +75,25 @@ constraint][language version] lower bound to 3.9 or greater (`sdk: '^3.9.0'`).
   Container<dynamic>().value('Invocation with missing runtime checks!');
   ```
 
+#### Dart CLI and Dart VM
+
+- The Dart CLI and Dart VM have been split into two seperate executables.
+
+  The Dart CLI tool has been split out of the VM into it's own embedder which
+  runs in AOT mode. The pure Dart VM executable is called `dartvm` and
+  has no Dart CLI functionality in it
+
+  The Dart CLI executable parses the CLI commands and invokes the rest
+  of the AOT tools in the same process, for the 'run' and 'test'
+  commands it execs a process which runs `dartvm`
+
+  `dart hello.dart` execs the `dartvm` process and runs 'hello.dart'
+
+  The Dart CLI is not generated for ia32 as we are not shipping a
+  Dart SDK for ia32 anymore (support to execute the `dartvm` for ia32
+  architecture is retained)
+
+
 #### Pub
 
 - Git dependencies can now be version-solved based on git tags.

build/dart/dart_action.gni

@@ -10,15 +10,15 @@ import("../executable_suffix.gni")
 # - prebuilt_dart_action()
 #   Runs Dart scripts using the downloaded prebuilt Dart SDK. This is the
 #   preferred method of running Dart code during the build as it is much
-#   faster than using dart_action() in debug and cross builds.
+#   faster than using dartvm_action() in debug and cross builds.
 #   However, prebuilt_dart_action() should *not* be used to generate snapshots.
 #
 # - prebuilt_dartaotruntime_action()
 #   Runs Dart AOT snapshots using the downloaded prebuilt Dart SDK. This is the
 #   preferred method of running Dart AOT code during the build as it avoids
 #   the multiple layers of code that prebuilt_dart_action goes through.
 #
-# - dart_action()
+# - dartvm_action()
 #   Runs Dart scripts using the binary built for runtime/bin:dart using the
 #   host toolchain. It should only be used when an artifact agreeing exactly
 #   with the version of the Dart VM being built must be produced, for example
@@ -350,10 +350,10 @@ template("_built_tool_action") {
 #    outputs
 #    testonly
 #    visibility
-template("dart_action") {
+template("dartvm_action") {
   assert(defined(invoker.script), "script must be defined for $target_name")
   _built_tool_action(target_name) {
-    tool = "$_dart_root/runtime/bin:dart"
+    tool = "$_dart_root/runtime/bin:dartvm"
     forward_variables_from(invoker,
                            [
                              "args",

build/rbe/rewrapper_dart.py

@@ -269,6 +269,9 @@ def parse_rewrapper(self):
             elif arg.endswith('/dart'):
                 self.dart_subdir = os.path.dirname(arg)
                 return self.parse_dart()
+            elif arg.endswith('/dartvm'):
+                self.dart_subdir = os.path.dirname(arg)
+                return self.parse_dart()
             elif arg.endswith('/dartaotruntime'):
                 self.dart_subdir = os.path.dirname(arg)
                 return self.parse_dartaotruntime()

pkg/analysis_server/integration_test/analysis/get_errors_non_standard_sdk_test.dart

@@ -27,6 +27,11 @@ class AnalysisDomainGetErrorsTest
       return path.join(sdkPath, 'bin', name);
     }
 
+    String executableVmFilePathIn(String sdkPath) {
+      var name = Platform.isWindows ? 'dartvm.exe' : 'dartvm';
+      return path.join(sdkPath, 'bin', name);
+    }
+
     String serverSnapshotPathIn(String sdkPath) {
       return path.join(
         sdkPath,
@@ -50,6 +55,10 @@ class AnalysisDomainGetErrorsTest
       executableFilePathIn(standardSdkPath),
     ).copySync(executableFilePathIn(sdkPath));
 
+    File(
+      executableVmFilePathIn(standardSdkPath),
+    ).copySync(executableVmFilePathIn(sdkPath));
+
     File(
       serverSnapshotPathIn(standardSdkPath),
     ).copySync(serverSnapshotPathIn(sdkPath));

pkg/dartdev/lib/src/commands/compile.dart

@@ -98,19 +98,12 @@ class CompileJSCommand extends CompileSubcommandCommand {
     }
     final args = argResults!;
     var snapshot = sdk.dart2jsAotSnapshot;
-    var script = sdk.dartAotRuntime;
-    var useExecProcess = true;
     if (!Sdk.checkArtifactExists(snapshot, logError: false)) {
-      // AOT snapshots cannot be generated on IA32, so we need this fallback
-      // branch until support for IA32 is dropped (https://dartbug.com/49969).
-      script = sdk.dart2jsSnapshot;
-      if (!Sdk.checkArtifactExists(script)) {
-        return genericErrorExitCode;
-      }
-      useExecProcess = false;
+      log.stderr('Error: JS compilation failed');
+      log.stderr('Unable to find $snapshot');
+      return compileErrorExitCode;
     }
     final dart2jsCommand = [
-      if (useExecProcess) snapshot,
       '--libraries-spec=${sdk.librariesJson}',
       '--cfe-invocation-modes=compile',
       '--invoker=dart_cli',
@@ -119,10 +112,10 @@ class CompileJSCommand extends CompileSubcommandCommand {
     ];
     try {
       VmInteropHandler.run(
-        script,
+        snapshot,
         dart2jsCommand,
         packageConfigOverride: null,
-        useExecProcess: useExecProcess,
+        useExecProcess: false,
       );
       return 0;
     } catch (e, st) {
@@ -163,28 +156,21 @@ class CompileDDCCommand extends CompileSubcommandCommand {
     }
     final args = argResults!;
     var snapshot = sdk.ddcAotSnapshot;
-    var script = sdk.dartAotRuntime;
-    var useExecProcess = true;
     if (!Sdk.checkArtifactExists(snapshot, logError: false)) {
-      // AOT snapshots cannot be generated on IA32, so we need this fallback
-      // branch until support for IA32 is dropped (https://dartbug.com/49969).
-      script = sdk.ddcSnapshot;
-      if (!Sdk.checkArtifactExists(script)) {
-        return genericErrorExitCode;
-      }
-      useExecProcess = false;
+      log.stderr('Error: JS compilation failed');
+      log.stderr('Unable to find $snapshot');
+      return compileErrorExitCode;
     }
     final ddcCommand = <String>[
-      if (useExecProcess) snapshot,
       // Add the remaining arguments.
       if (args.rest.isNotEmpty) ...args.rest.sublist(0),
     ];
     try {
       VmInteropHandler.run(
-        script,
+        snapshot,
         ddcCommand,
         packageConfigOverride: null,
-        useExecProcess: useExecProcess,
+        useExecProcess: false,
       );
       return 0;
     } catch (e, st) {
@@ -459,7 +445,7 @@ class CompileJitSnapshotCommand extends CompileSubcommandCommand {
 
     log.stdout('Compiling $sourcePath to jit-snapshot file $outputFile.');
     // TODO(bkonyi): perform compilation in same process.
-    return await runProcess([sdk.dart, ...buildArgs]);
+    return await runProcess([sdk.dartvm, ...buildArgs]);
   }
 }
 

pkg/dartdev/lib/src/commands/dart_mcp_server.dart

@@ -51,12 +51,11 @@ A stdio based Model Context Protocol (MCP) server to aid in Dart and Flutter dev
     }
     try {
       VmInteropHandler.run(
-        sdk.dartAotRuntime,
+        sdk.dartMCPServerAotSnapshot,
         [
-          sdk.dartMCPServerAotSnapshot,
           ...forwardedArgs,
         ],
-        useExecProcess: true,
+        useExecProcess: false,
       );
       return 0;
     } catch (e, st) {

pkg/dartdev/lib/src/commands/development_service.dart

@@ -3,7 +3,6 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:io';
 
 import 'package:dds/src/arg_parser.dart';
 import 'package:path/path.dart';
@@ -37,38 +36,22 @@ class DevelopmentServiceCommand extends DartdevCommand {
   Future<int> run() async {
     final sdkDir = dirname(sdk.dart);
     final fullSdk = sdkDir.endsWith('bin');
-    var script = fullSdk
-        ? sdk.dartAotRuntime
-        : absolute(sdkDir, 'dartaotruntime${Platform.isWindows ? '.exe' : ''}');
     var snapshot = fullSdk
         ? sdk.ddsAotSnapshot
         : absolute(sdkDir, 'dds_aot.dart.snapshot');
-    var useExecProcess = true;
     final args = argResults!.arguments;
 
     if (!Sdk.checkArtifactExists(snapshot, logError: false)) {
-      // On ia32 platforms we do not have an AOT snapshot and so we need
-      // to run the JIT snapshot.
-      useExecProcess = false;
-      script =
-          fullSdk ? sdk.ddsSnapshot : absolute(sdkDir, 'dds.dart.snapshot');
-      if (!Sdk.checkArtifactExists(script, logError: false)) {
-        log.stderr('Error: launching development server failed : '
-            'Unable to find snapshot for the development server');
-        return 255;
-      }
+      log.stderr('Error: launching development server failed : '
+                 'Unable to find snapshot for the development server');
+      return 255;
     }
-    final ddsCommand = [
-      if (useExecProcess) snapshot,
-      // Add the remaining args.
-      if (args.isNotEmpty) ...args,
-    ];
     try {
       VmInteropHandler.run(
-        script,
-        ddsCommand,
+        snapshot,
+        args,
         packageConfigOverride: null,
-        useExecProcess: useExecProcess,
+        useExecProcess: false,
       );
       return 0;
     } catch (e, st) {

pkg/dartdev/lib/src/commands/language_server.dart

@@ -54,34 +54,35 @@ For more information about the server's capabilities and configuration, see:
       args = [...args, '--$protocol=$lsp'];
     }
     try {
+      var script = sdk.analysisServerAotSnapshot;
+      var useExec = false;
       if (argResults!.flag(useAotSnapshotFlag)) {
-        if (!Sdk.checkArtifactExists(sdk.dartAotRuntime)) {
+        if (!Sdk.checkArtifactExists(sdk.analysisServerAotSnapshot)) {
+          log.stderr('Error: launching language analysis server failed');
+          log.stderr('${sdk.analysisServerAotSnapshot} not found');
           return _genericErrorExitCode;
         }
         args = [...args];
         args.remove('--$useAotSnapshotFlag');
-        VmInteropHandler.run(
-          sdk.dartAotRuntime,
-          [sdk.analysisServerAotSnapshot, ...args],
-          useExecProcess: true,
-        );
       } else {
         args = [...args];
         args.remove('--no-$useAotSnapshotFlag');
-        VmInteropHandler.run(
-          sdk.analysisServerSnapshot,
-          args,
-          useExecProcess: false,
-        );
+        script = sdk.analysisServerSnapshot;
+        useExec = true;
       }
+      VmInteropHandler.run(
+        script,
+        args,
+        useExecProcess: useExec,
+      );
       return 0;
     } catch (e, st) {
       log.stderr('Error: launching language analysis server failed');
       log.stderr(e.toString());
       if (verbose) {
         log.stderr(st.toString());
       }
-      return 255;
+      return _genericErrorExitCode;
     }
   }