[vm] Add option to write a heap snapshot on OOM.

TEST=ci
Bug: #61036
Change-Id: I3f9ac67179ba54453f18695853afa78501d587a5
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/439344
Reviewed-by: Alexander Aprelev <[email protected]>
Commit-Queue: Ryan Macnak <[email protected]>

Author: rmacnak-google

runtime/tests/vm/dart/heap_snapshot_on_oom_test.dart

@@ -0,0 +1,44 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import "dart:io";
+import "package:expect/expect.dart";
+import "use_flag_test_helper.dart";
+
+main(List<String> arguments) async {
+  if (arguments.contains("--testee")) {
+    var x = [];
+    while (true) {
+      x = [x];
+    }
+    return;
+  }
+
+  // Snapshot write omitted from product mode.
+  if (const bool.fromEnvironment("dart.vm.product")) return;
+
+  await withTempDir("heap_snapshot_on_oom_test", (String dir) async {
+    var exec = Platform.executable;
+    var args = [
+      ...Platform.executableArguments,
+      "--heap_snapshot_on_oom=$dir/oom.heapsnapshot",
+      "--old_gen_heap_size=50", // MB
+      Platform.script.toFilePath(),
+      "--testee",
+    ];
+    print("+ $exec ${args.join(' ')}");
+
+    var result = Process.runSync(exec, args);
+    print("Command stdout:");
+    print(result.stdout);
+    print("Command stderr:");
+    print(result.stderr);
+
+    Expect.equals(255, result.exitCode);
+    Expect.contains("Out of Memory", result.stderr);
+
+    Expect.isTrue(await File("$dir/oom.heapsnapshot").exists());
+    Expect.isTrue(await File("$dir/oom.heapsnapshot").length() > 0);
+  });
+}

runtime/vm/heap/heap.cc

@@ -21,6 +21,7 @@
 #include "vm/isolate.h"
 #include "vm/lockers.h"
 #include "vm/object.h"
+#include "vm/object_graph.h"
 #include "vm/object_set.h"
 #include "vm/os.h"
 #include "vm/raw_object.h"
@@ -40,6 +41,13 @@ DEFINE_FLAG(bool,
             disable_heap_verification,
             false,
             "Explicitly disable heap verification.");
+#if defined(DART_ENABLE_HEAP_SNAPSHOT_WRITER)
+DEFINE_FLAG(charp,
+            heap_snapshot_on_oom,
+            nullptr,
+            "Write a heap snapshot after encountering an Out of Memory error. "
+            "Best used with a reduced --old_gen_heap_size");
+#endif
 
 Heap::Heap(IsolateGroup* isolate_group,
            bool is_vm_isolate,
@@ -184,6 +192,20 @@ uword Heap::AllocateOld(Thread* thread, intptr_t size, bool is_exec) {
   OS::PrintErr("Exhausted heap space, trying to allocate %" Pd " bytes.\n",
                size);
   isolate_group_->set_has_seen_oom(true);
+#if defined(DART_ENABLE_HEAP_SNAPSHOT_WRITER)
+  if (FLAG_heap_snapshot_on_oom != nullptr) {
+    bool successful = false;
+    {
+      FileHeapSnapshotWriter file_writer(thread, FLAG_heap_snapshot_on_oom,
+                                         &successful);
+      HeapSnapshotWriter writer(thread, &file_writer);
+      writer.Write();
+    }
+    if (!successful) {
+      OS::PrintErr("Failed to write heapsnapshot\n");
+    }
+  }
+#endif
   return 0;
 }