[vm/shared] Update mutator count as entering/exiting mutator thread.

Running dart code on mutator thread outside of isolate still should follow limits on number of concurrent mutators in the vm.
Also this fixes inconsistency when entering isolate group as mutator, where original worker was marked as blocked while thread was actually running dart code. That inconsitency showed up as sporadic crash on isolate_group_shared_init_test with increased number of spawned isolates.

TEST=isolate_group_shared_init_test
BUG=#60877

Change-Id: I8917be903bb5517940dad3e5dab143d5ed5fdf79
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/432900
Reviewed-by: Ryan Macnak <[email protected]>
Commit-Queue: Alexander Aprelev <[email protected]>

Author: aam

runtime/vm/isolate.cc

@@ -647,14 +647,11 @@ void IsolateGroup::IncreaseMutatorCount(Thread* thread,
   }
 }
 
-void IsolateGroup::DecreaseMutatorCount(Isolate* mutator, bool is_nested_exit) {
-  ASSERT(mutator->group() == this);
-
+void IsolateGroup::DecreaseMutatorCount(bool is_nested_exit) {
   // If the mutator thread has an active stack and runs on our thread pool we
   // will mark the worker as blocked, thereby possibly spawning a new worker for
   // pending tasks (if there are any).
   if (is_nested_exit) {
-    ASSERT(mutator->mutator_thread() != nullptr);
     thread_pool()->MarkCurrentWorkerAsBlocked();
   }
 

runtime/vm/isolate.h

@@ -548,7 +548,7 @@ class IsolateGroup : public IntrusiveDListEntry<IsolateGroup> {
   void IncreaseMutatorCount(Thread* thread,
                             bool is_nested_reenter,
                             bool was_stolen);
-  void DecreaseMutatorCount(Isolate* mutator, bool is_nested_exit);
+  void DecreaseMutatorCount(bool is_nested_exit);
   NO_SANITIZE_THREAD
   intptr_t MutatorCount() const { return active_mutators_; }
 

runtime/vm/thread.cc

@@ -489,7 +489,7 @@ void Thread::ExitIsolate(bool isolate_shutdown) {
   // occupying a mutator again (decreases its max size).
   ASSERT(!(isolate_shutdown && is_nested_exit));
   if (!(is_nested_exit && thread->OwnsSafepoint())) {
-    group->DecreaseMutatorCount(isolate, is_nested_exit);
+    group->DecreaseMutatorCount(is_nested_exit);
   }
 }
 
@@ -538,6 +538,10 @@ void Thread::EnterIsolateGroupAsMutator(IsolateGroup* isolate_group,
   thread->SetStackLimit(OSThread::Current()->overflow_stack_limit());
 #endif
 
+  isolate_group->IncreaseMutatorCount(/*thread=*/thread,
+                                      /*is_nested_reenter=*/false,
+                                      /*was_stolen=*/true);
+
   thread->AssertDartMutatorInvariants();
 }
 
@@ -551,6 +555,7 @@ void Thread::ExitIsolateGroupAsMutator(bool bypass_safepoint) {
   thread->ResetMutatorState();
   thread->ClearStackLimit();
   SuspendThreadInternal(thread, VMTag::kInvalidTagId);
+  thread->isolate_group()->DecreaseMutatorCount(/*is_nested_exit=*/true);
   FreeActiveThread(thread, /*isolate=*/nullptr, bypass_safepoint);
 }
 

tests/ffi/isolate_group_shared_init_test.dart

@@ -42,7 +42,7 @@ late final String foo = () {
 }();
 
 testInitStrings() async {
-  const int nWorkers = 20;
+  const int nWorkers = 100;
   mutex = Mutex();
   final rp = ReceivePort();
   final rpExitAndErrors = ReceivePort()
@@ -78,7 +78,7 @@ testInitStrings() async {
 }
 
 testInitThrows() async {
-  const int nWorkers = 20;
+  const int nWorkers = 100;
   final rp = ReceivePort();
   int exitCounter = 0;
   final completer = Completer();