[vm] Fix deadlock in NativeCallable.listener

Switch all lockers of FfiCallbackMetadata::lock_ in FfiCallbackMetadata
from MutexLocker to SafepointMutexLocker.

The new test deadlocks on patchset 2, before the fix:
https://logs.chromium.org/logs/dart/buildbucket/cr-buildbucket/8706690957454603105/+/u/test_results/new_test_failures__logs_

And succeeds on patchset 4, after the fix.

I found that switching the MutexLocker in DLRT_GetFfiCallbackMetadata to
a SafepointMutexLocker caused other NativeCallable.listener tests to
fail, so I've left that one unchanged:
https://logs.chromium.org/logs/dart/buildbucket/cr-buildbucket/8706685486083640001/+/u/test_results/new_test_failures__logs_

Change-Id: I3e48e05d56806488754751b23db27f208f9d396d
Fixes: #61272
TEST=tests/ffi/many_listener_callbacks_test.dart
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/445000
Commit-Queue: Liam Appelbe <[email protected]>
Reviewed-by: Slava Egorov <[email protected]>

Author: liamappelbe

runtime/bin/ffi_test/ffi_test_functions.cc

@@ -1332,6 +1332,29 @@ DART_EXPORT void CallFunctionOnNewThreadBlocking(int64_t response_id,
   thread.join();
 }
 
+struct RepeatingThread {
+  std::thread thread;
+  std::atomic<bool> run;
+};
+
+DART_EXPORT void* CallFunctionOnNewThreadRepeatedly(void (*fn)(int64_t)) {
+  RepeatingThread* repeater = new RepeatingThread();
+  repeater->run.store(true);
+  repeater->thread = std::thread([fn, repeater]() {
+    for (int64_t i = 0; repeater->run.load(); ++i) {
+      fn(i);
+    }
+  });
+  return repeater;
+}
+
+DART_EXPORT void CallFunctionOnNewThreadStop(void* ptr) {
+  RepeatingThread* repeater = reinterpret_cast<RepeatingThread*>(ptr);
+  repeater->run.store(false);
+  repeater->thread.join();
+  delete repeater;
+}
+
 #if defined(__linux__)
 struct Data {
   void (*fn)(int64_t, int32_t);

runtime/vm/ffi_callback_metadata.cc

@@ -251,7 +251,7 @@ FfiCallbackMetadata::Trampoline FfiCallbackMetadata::CreateMetadataEntry(
     uword target_entry_point,
     uint64_t context,
     MetadataEntry** list_head) {
-  MutexLocker locker(&lock_);
+  SafepointMutexLocker locker(&lock_);
   EnsureFreeListNotEmptyLocked();
   ASSERT(free_list_head_ != nullptr);
   MetadataEntry* entry = free_list_head_;
@@ -305,7 +305,7 @@ void FfiCallbackMetadata::DeleteCallbackLocked(MetadataEntry* entry) {
 }
 
 void FfiCallbackMetadata::DeleteAllCallbacks(MetadataEntry** list_head) {
-  MutexLocker locker(&lock_);
+  SafepointMutexLocker locker(&lock_);
   for (MetadataEntry* entry = *list_head; entry != nullptr;) {
     MetadataEntry* next = entry->list_next();
     DeleteCallbackLocked(entry);
@@ -316,7 +316,7 @@ void FfiCallbackMetadata::DeleteAllCallbacks(MetadataEntry** list_head) {
 
 void FfiCallbackMetadata::DeleteCallback(Trampoline trampoline,
                                          MetadataEntry** list_head) {
-  MutexLocker locker(&lock_);
+  SafepointMutexLocker locker(&lock_);
   auto* entry = MetadataEntryOfTrampoline(trampoline);
   ASSERT(entry->metadata()->IsLive());
   auto* prev = entry->list_prev_;

tests/ffi/many_listener_callbacks_test.dart

@@ -0,0 +1,67 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// Tests that interleaved creation and calling of many NativeCallable.listener
+// callbacks does not deadlock.
+// Regression test for https://github.com/dart-lang/sdk/issues/61272
+//
+// VMOptions=
+// VMOptions=--use-slow-path
+// VMOptions=--use-slow-path --stacktrace-every=100
+// VMOptions=--dwarf_stack_traces --no-retain_function_objects --no-retain_code_objects
+// VMOptions=--test_il_serialization
+// VMOptions=--profiler --profile_vm=true
+// VMOptions=--profiler --profile_vm=false
+// SharedObjects=ffi_test_functions
+
+import 'dart:async';
+import 'dart:ffi';
+
+import 'dylib_utils.dart';
+
+typedef FnStartNativeType = Pointer Function(Pointer);
+typedef FnStartType = Pointer Function(Pointer);
+typedef FnStopNativeType = Void Function(Pointer);
+typedef FnStopType = void Function(Pointer);
+
+late final FnStartType callFunctionOnNewThreadRepeatedly;
+late final FnStopType callFunctionOnNewThreadStop;
+
+void main() async {
+  final ffiTestFunctions = dlopenPlatformSpecific('ffi_test_functions');
+  callFunctionOnNewThreadRepeatedly = ffiTestFunctions
+      .lookupFunction<FnStartNativeType, FnStartType>(
+        'CallFunctionOnNewThreadRepeatedly',
+      );
+  callFunctionOnNewThreadStop = ffiTestFunctions
+      .lookupFunction<FnStopNativeType, FnStopType>(
+        'CallFunctionOnNewThreadStop',
+      );
+
+  final repeaters = <(Pointer, NativeCallable)>[];
+  void spawn(void Function(int) callback) {
+    final listener = NativeCallable<Void Function(Int32)>.listener(callback);
+    final repeater = callFunctionOnNewThreadRepeatedly(listener.nativeFunction);
+    repeaters.add((repeater, listener));
+  }
+
+  // Spawn one repeater, wait for it to start running, then spawn 30 more.
+  final firstIsRunning = Completer<void>();
+  spawn((int n) {
+    if (n == 10) firstIsRunning.complete();
+  });
+
+  await firstIsRunning.future;
+
+  for (var i = 0; i < 30; ++i) {
+    spawn((int n) {});
+  }
+
+  await Future.delayed(Duration(seconds: 3));
+
+  for (final (repeater, listener) in repeaters) {
+    callFunctionOnNewThreadStop(repeater);
+    listener.close();
+  }
+}