[vm] Make out-of-memory scenarios a bit more robust.

 - Don't scavenge when there is no OOM reservation, so refilling the OOM reservation will get the first go once free space is available instead it being consumed by promotions.
 - Increase the desired free space in new space needed to continue scavenging. Since new space became paged, 1 KB / 64 pages only guarentees the ability to allocate a 16 byte object.
 - Stop issuing stress test reloads after an OOM is encountered.
 - Add a HANDLESCOPE to reload to make memory collectible sooner after an error.

TEST=vm/out_of_memory
Bug: #48920
Bug: #53455
Bug: #43642
Change-Id: Ife8bed52e341648a66f78d55bfa60b3944be5d46
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/422324
Commit-Queue: Ryan Macnak <[email protected]>
Reviewed-by: Alexander Aprelev <[email protected]>

Author: rmacnak-google

runtime/vm/heap/heap.cc

@@ -83,7 +83,8 @@ uword Heap::AllocateNew(Thread* thread, intptr_t size) {
   if (LIKELY(addr != 0)) {
     return addr;
   }
-  if (!assume_scavenge_will_fail_ && !thread->force_growth()) {
+  if (!assume_scavenge_will_fail_ && !thread->force_growth() &&
+      old_space_.HasReservation()) {
     GcSafepointOperationScope safepoint_operation(thread);
 
     // Another thread may have won the race to the safepoint and performed a GC
@@ -174,6 +175,7 @@ uword Heap::AllocateOld(Thread* thread, intptr_t size, bool is_exec) {
   // Give up allocating this object.
   OS::PrintErr("Exhausted heap space, trying to allocate %" Pd " bytes.\n",
                size);
+  isolate_group_->set_has_seen_oom(true);
   return 0;
 }
 

runtime/vm/heap/pages.h

@@ -168,6 +168,7 @@ class PageSpace {
     return AllocateSnapshotLockedSlow(freelist, size);
   }
 
+  bool HasReservation() { return oom_reservation_ != nullptr; }
   void TryReleaseReservation();
   bool MarkReservation();
   void TryReserveForOOM();
@@ -449,7 +450,10 @@ class PageSpace {
     kDataFreelist = 1,
   };
   FreeList* freelists_;
+
   static constexpr intptr_t kOOMReservationSize = 32 * KB;
+  static_assert(kOOMReservationSize < kAllocatablePageSize,
+                "OOM reservation should not go to a large page");
   FreeListElement* oom_reservation_ = nullptr;
 
   // Use ExclusivePageIterator for safe access to these.

runtime/vm/heap/scavenger.cc

@@ -1932,7 +1932,7 @@ void Scavenger::Scavenge(Thread* thread, GCType type, GCReason reason) {
     ReverseScavenge(&from);
     bytes_promoted = 0;
   } else {
-    if ((ThresholdInWords() - UsedInWords()) < KBInWords) {
+    if ((ThresholdInWords() - UsedInWords()) < 32 * KBInWords) {
       // Don't scavenge again until the next old-space GC has occurred. Prevents
       // performing one scavenge per allocation as the heap limit is approached.
       heap_->assume_scavenge_will_fail_ = true;

runtime/vm/isolate.h

@@ -481,6 +481,13 @@ class IsolateGroup : public IntrusiveDListEntry<IsolateGroup> {
 #endif  // !defined(DART_PRECOMPILED_RUNTIME)
 #endif  // !defined(PRODUCT)
 
+  bool has_seen_oom() const {
+    return HasSeenOOMBit::decode(isolate_group_flags_);
+  }
+  void set_has_seen_oom(bool value) {
+    isolate_group_flags_ = HasSeenOOMBit::update(value, isolate_group_flags_);
+  }
+
 #if defined(PRODUCT)
   void set_use_osr(bool use_osr) { ASSERT(!use_osr); }
 #else   // defined(PRODUCT)
@@ -801,6 +808,7 @@ class IsolateGroup : public IntrusiveDListEntry<IsolateGroup> {
   V(AllClassesFinalized)                                                       \
   V(EnableAsserts)                                                             \
   V(HasAttemptedReload)                                                        \
+  V(HasSeenOOM)                                                                \
   V(RemappingCids)                                                             \
   V(ShouldLoadVmService)                                                       \
   V(Obfuscate)                                                                 \

runtime/vm/isolate_reload.cc

@@ -1220,7 +1220,8 @@ ObjectPtr ProgramReloadContext::ReloadPhase2LoadKernel(
     const String& root_lib_url) {
   Thread* thread = Thread::Current();
 
-  LongJumpScope jump;
+  HANDLESCOPE(thread);
+  LongJumpScope jump(thread);
   if (DART_SETJMP(*jump.Set()) == 0) {
     const Object& tmp = kernel::KernelLoader::LoadEntireProgram(program);
     if (tmp.IsError()) {

runtime/vm/runtime_entry.cc

@@ -3216,7 +3216,8 @@ static void HandleStackOverflowTestCases(Thread* thread) {
         do_gc = true;
       }
       if ((isolate_reload_every > 0) && (count % isolate_reload_every) == 0) {
-        do_reload = isolate->group()->CanReload();
+        do_reload =
+            isolate_group->CanReload() && !isolate_group->has_seen_oom();
       }
     }
   }
@@ -3275,7 +3276,7 @@ static void HandleStackOverflowTestCases(Thread* thread) {
     JSONStream js;
     const bool success =
         isolate_group->ReloadSources(&js, /*force_reload=*/true, script_uri);
-    if (!success && !Dart::IsShuttingDown()) {
+    if (!success && !Dart::IsShuttingDown() && !isolate_group->has_seen_oom()) {
       FATAL("*** Isolate reload failed:\n%s\n", js.ToCString());
     }
   }