Reland "[VM/Service] Record timeline events representing completed microtasks"

This is a reland of commit 22021cc

I showed some excerpts of Golem results to @mraleph for varying targets
and constant CPU, patch A, and patch B. Those constants being:
  CPU: Intel Xeon
  patch A: revision=115869&patch=20143
  patch B: revision=115876
(example results:
https://golem.corp.goog/Comparison?repository=dart#targetA%3Ddart%3BmachineTypeA%3Dlinux-x64%3BrevisionA%3D115869%3BpatchA%3Dderekx-Reland---VM%2FService--Record-timeline-events-representing-completed-microtasks--4%3BtargetB%3Ddart%3BmachineTypeB%3Dlinux-x64%3BrevisionB%3D115876%3BpatchB%3DNone)

He said that this change could be relanded as long as the following
"RunTime as Score" regressions of the "dart" target were noted in the
commit message:
  - StreamSingleSubTest: -3.907% (-0.8 noise)
  - ScheduleMicrotaskTest: -12.67% (-0.6 noise)

TEST=pkg/vm_service/test/timeline_events_for_completed_microtasks_test

Original change's description:
> [VM/Service] Record timeline events representing completed microtasks
>
> TEST=pkg/vm_service/test/timeline_events_for_completed_microtasks_test
>
> CoreLibraryReviewExempt: This CL does not include any core library API
> changes, it only modifies the implementation of microtasks (by
> instrumenting them).
> Change-Id: I54d886db9519c73f9e3218a9cc1c46bc9fe9acc3
> Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/420221
> Commit-Queue: Derek Xu <[email protected]>
> Reviewed-by: Ben Konyi <[email protected]>

CoreLibraryReviewExempt: This CL does not include any core library API
changes, it only modifies the implementation of microtasks (by
instrumenting them).
Change-Id: Ia7741a9bb9ab948e8d5fff1cf9abe9915e247d81
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/423921
Reviewed-by: Ben Konyi <[email protected]>
Commit-Queue: Derek Xu <[email protected]>

Author: derekxu16

pkg/vm_service/test/common/service_test_common.dart

@@ -5,11 +5,13 @@
 library service_test_common;
 
 import 'dart:async';
+import 'dart:collection' show HashMap;
 import 'dart:typed_data';
 
 import 'package:path/path.dart' as p;
 import 'package:test/test.dart';
 import 'package:vm_service/vm_service.dart';
+import 'package:vm_service_protos/vm_service_protos.dart' show DebugAnnotation;
 
 typedef IsolateTest = Future<void> Function(
   VmService service,
@@ -914,3 +916,19 @@ IsolateTest testExpressionEvaluationAndAvailableVariables(
     }
   };
 }
+
+Map<String, String> mapFromListOfDebugAnnotations(
+  List<DebugAnnotation> debugAnnotations,
+) {
+  return HashMap.fromEntries(
+    debugAnnotations.map((a) {
+      if (a.hasStringValue()) {
+        return MapEntry(a.name, a.stringValue);
+      } else if (a.hasLegacyJsonValue()) {
+        return MapEntry(a.name, a.legacyJsonValue);
+      } else {
+        throw 'We should not be writing annotations without values';
+      }
+    }),
+  );
+}

pkg/vm_service/test/get_perfetto_vm_timeline_rpc_test.dart

@@ -5,7 +5,6 @@
 // VMOptions=
 // VMOptions=--intern_strings_when_writing_perfetto_timeline
 
-import 'dart:collection';
 import 'dart:convert';
 import 'dart:developer';
 import 'dart:io' show Platform;
@@ -14,6 +13,7 @@ import 'package:test/test.dart';
 import 'package:vm_service/vm_service.dart' hide Timeline;
 import 'package:vm_service_protos/vm_service_protos.dart';
 
+import 'common/service_test_common.dart' show mapFromListOfDebugAnnotations;
 import 'common/test_helper.dart';
 
 void primeTimeline() {
@@ -143,22 +143,6 @@ List<TrackEvent> extractTrackEventsFromTracePackets(
   return result;
 }
 
-Map<String, String> mapFromListOfDebugAnnotations(
-  List<DebugAnnotation> debugAnnotations,
-) {
-  return HashMap.fromEntries(
-    debugAnnotations.map((a) {
-      if (a.hasStringValue()) {
-        return MapEntry(a.name, a.stringValue);
-      } else if (a.hasLegacyJsonValue()) {
-        return MapEntry(a.name, a.legacyJsonValue);
-      } else {
-        throw 'We should not be writing annotations without values';
-      }
-    }),
-  );
-}
-
 void checkThatAllEventsHaveIsolateNumbers(Iterable<TrackEvent> events) {
   for (final event in events) {
     final debugAnnotations =

pkg/vm_service/test/timeline_events_for_completed_microtasks_test.dart

@@ -0,0 +1,64 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+
+import 'package:test/test.dart';
+import 'package:vm_service/vm_service.dart' hide Timeline;
+import 'package:vm_service_protos/vm_service_protos.dart';
+
+import 'common/service_test_common.dart' show mapFromListOfDebugAnnotations;
+import 'common/test_helper.dart';
+
+const String shortFile = 'timeline_events_for_completed_microtasks_test.dart';
+
+void primeTimeline() {
+  for (int i = 0; i < 5; i++) {
+    scheduleMicrotask(() {});
+  }
+}
+
+Iterable<TrackEvent> extractTrackEventsFromTracePackets(
+  List<TracePacket> packets,
+) =>
+    packets
+        .where((packet) => packet.hasTrackEvent())
+        .map((packet) => packet.trackEvent);
+
+final tests = <IsolateTest>[
+  (VmService service, IsolateRef isolateRef) async {
+    final result = await service.getPerfettoVMTimeline();
+
+    final trace = Trace.fromBuffer(base64Decode(result.trace!));
+    final packets = trace.packet;
+    final mainIsolateMicrotaskEvents =
+        extractTrackEventsFromTracePackets(packets)
+            .where((event) => event.name == 'Microtask')
+            .where((event) {
+      final debugAnnotations =
+          mapFromListOfDebugAnnotations(event.debugAnnotations);
+      return debugAnnotations['isolateId'] == isolateRef.id;
+    });
+    expect(mainIsolateMicrotaskEvents.length, greaterThanOrEqualTo(5));
+
+    for (final event in mainIsolateMicrotaskEvents) {
+      final debugAnnotations =
+          mapFromListOfDebugAnnotations(event.debugAnnotations);
+      expect(debugAnnotations['microtaskId'], isNotNull);
+      expect(
+        debugAnnotations['stack trace captured when microtask was enqueued'],
+        contains(shortFile),
+      );
+    }
+  },
+];
+
+void main([args = const <String>[]]) => runIsolateTests(
+      args,
+      tests,
+      shortFile,
+      testeeBefore: primeTimeline,
+      extraArgs: ['--profile-microtasks', '--timeline-streams=Microtask'],
+    );

runtime/bin/dart_embedder_api_impl.cc

@@ -75,8 +75,9 @@ Dart_Isolate CreateKernelServiceIsolate(const IsolateCreationData& data,
     Dart_ShutdownIsolate();
     return nullptr;
   }
-  result = bin::DartUtils::PrepareForScriptLoading(/*is_service_isolate=*/false,
-                                                   /*trace_loading=*/false);
+  result = bin::DartUtils::PrepareForScriptLoading(
+      /*is_service_isolate=*/false,
+      /*trace_loading=*/false, /*flag_profile_microtasks=*/false);
   Dart_ExitScope();
   Dart_ExitIsolate();
   return kernel_isolate;

runtime/bin/dartutils.cc

@@ -529,7 +529,31 @@ Dart_Handle DartUtils::PrepareCoreLibrary(Dart_Handle core_lib,
 }
 
 Dart_Handle DartUtils::PrepareAsyncLibrary(Dart_Handle async_lib,
-                                           Dart_Handle isolate_lib) {
+                                           Dart_Handle isolate_lib,
+                                           bool flag_profile_microtasks) {
+#if !defined(PRODUCT)
+  if (flag_profile_microtasks) {
+    Dart_Handle microtask_mirror_queue_type_name =
+        Dart_NewStringFromCString("_MicrotaskMirrorQueue");
+    RETURN_IF_ERROR(microtask_mirror_queue_type_name);
+
+    Dart_Handle microtask_mirror_queue_type =
+        Dart_GetNonNullableType(async_lib, microtask_mirror_queue_type_name,
+                                /*number_of_type_arguments=*/0,
+                                /*type_arguments=*/nullptr);
+    RETURN_IF_ERROR(microtask_mirror_queue_type);
+
+    Dart_Handle should_profile_microtasks_field_name =
+        Dart_NewStringFromCString("_shouldProfileMicrotasks");
+    RETURN_IF_ERROR(should_profile_microtasks_field_name);
+
+    Dart_Handle set_field_result =
+        Dart_SetField(microtask_mirror_queue_type,
+                      should_profile_microtasks_field_name, Dart_True());
+    RETURN_IF_ERROR(set_field_result);
+  }
+#endif  // !defined(PRODUCT)
+
   Dart_Handle schedule_immediate_closure =
       Dart_Invoke(isolate_lib, NewString("_getIsolateScheduleImmediateClosure"),
                   0, nullptr);
@@ -568,7 +592,8 @@ Dart_Handle DartUtils::SetupPackageConfig(const char* packages_config) {
 }
 
 Dart_Handle DartUtils::PrepareForScriptLoading(bool is_service_isolate,
-                                               bool trace_loading) {
+                                               bool trace_loading,
+                                               bool flag_profile_microtasks) {
   // First ensure all required libraries are available.
   Dart_Handle url = NewString(kCoreLibURL);
   RETURN_IF_ERROR(url);
@@ -610,7 +635,8 @@ Dart_Handle DartUtils::PrepareForScriptLoading(bool is_service_isolate,
                                  trace_loading);
   RETURN_IF_ERROR(result);
 
-  RETURN_IF_ERROR(PrepareAsyncLibrary(async_lib, isolate_lib));
+  RETURN_IF_ERROR(
+      PrepareAsyncLibrary(async_lib, isolate_lib, flag_profile_microtasks));
   RETURN_IF_ERROR(PrepareCoreLibrary(core_lib, io_lib, is_service_isolate));
   RETURN_IF_ERROR(PrepareIsolateLibrary(isolate_lib));
   RETURN_IF_ERROR(PrepareIOLibrary(io_lib));

runtime/bin/dartutils.h

@@ -178,7 +178,8 @@ class DartUtils {
   static Dart_Handle ReadStringFromFile(const char* filename);
   static Dart_Handle MakeUint8Array(const void* buffer, intptr_t length);
   static Dart_Handle PrepareForScriptLoading(bool is_service_isolate,
-                                             bool trace_loading);
+                                             bool trace_loading,
+                                             bool flag_profile_microtasks);
   static Dart_Handle SetupPackageConfig(const char* packages_file);
 
   static Dart_Handle SetupIOLibrary(const char* namespc_path,
@@ -316,7 +317,8 @@ class DartUtils {
                                         Dart_Handle io_lib,
                                         bool is_service_isolate);
   static Dart_Handle PrepareAsyncLibrary(Dart_Handle async_lib,
-                                         Dart_Handle isolate_lib);
+                                         Dart_Handle isolate_lib,
+                                         bool flag_profile_microtasks);
   static Dart_Handle PrepareIOLibrary(Dart_Handle io_lib);
   static Dart_Handle PrepareIsolateLibrary(Dart_Handle isolate_lib);
   static Dart_Handle PrepareCLILibrary(Dart_Handle cli_lib);

runtime/bin/main_impl.cc

@@ -161,7 +161,13 @@ static Dart_Handle SetupCoreLibraries(Dart_Isolate isolate,
   // Prepare builtin and other core libraries for use to resolve URIs.
   // Set up various closures, e.g: printing, timers etc.
   // Set up package configuration for URI resolution.
-  result = DartUtils::PrepareForScriptLoading(false, Options::trace_loading());
+#if defined(PRODUCT)
+  bool flag_profile_microtasks = false;
+#else
+  bool flag_profile_microtasks = Options::profile_microtasks();
+#endif  // defined(PRODUCT)
+  result = DartUtils::PrepareForScriptLoading(false, Options::trace_loading(),
+                                              flag_profile_microtasks);
   if (Dart_IsError(result)) return result;
 
   // Setup packages config if specified.

runtime/bin/main_options.cc

@@ -213,6 +213,9 @@ void Options::PrintUsage() {
 "      --timeline-streams=\"Compiler, Dart, GC, Microtask\"\n"
 "  This set is subject to change.\n"
 "  Please see these options for further documentation.\n"
+"--profile-microtasks\n"
+"  Record information about each microtask. Information about completed\n"
+"  microtasks will be written to the \"Microtask\" timeline stream.\n"
 #endif  // !defined(PRODUCT)
 "--version\n"
 "  Print the VM version.\n"

runtime/bin/main_options.h

@@ -58,6 +58,7 @@ namespace bin {
   V(no_serve_observatory, disable_observatory)                                 \
   V(serve_observatory, enable_observatory)                                     \
   V(print_dtd, print_dtd)                                                      \
+  V(profile_microtasks, profile_microtasks)                                    \
   /* The purpose of this flag is documented in */                              \
   /* pkg/dartdev/lib/src/commands/run.dart. */                                 \
   V(resident, resident)

runtime/bin/run_vm_tests.cc

@@ -255,7 +255,8 @@ static Dart_Isolate CreateIsolateAndSetup(const char* script_uri,
 
   bin::DartUtils::SetOriginalWorkingDirectory();
   Dart_Handle result = bin::DartUtils::PrepareForScriptLoading(
-      /*is_service_isolate=*/false, /*trace_loading=*/false);
+      /*is_service_isolate=*/false, /*trace_loading=*/false,
+      /*flag_profile_microtasks=*/false);
   CHECK_RESULT(result);
 
   // Setup kernel service as the main script for this isolate.