[vm/ffi] Error on invoking callback from leaf call

All callbacks flow via the single runtime entry, so we add a check
there.

TEST=tests/ffi/function_callbacks_leaf_test.dart

(The transition verification in
https://dart-review.googlesource.com/c/sdk/+/407021
doesn't catch the error. The callbacks already transitioned before
reaching the generated code in that CL.)

Closes: #60021
Change-Id: Ia6bf8af186c9c8c5a66cffa537d80421467330d4
Cq-Include-Trybots: dart/try:vm-aot-android-release-arm64c-try,vm-aot-android-release-arm_x64-try,vm-aot-asan-linux-release-x64-try,vm-aot-linux-debug-x64-try,vm-aot-linux-debug-x64c-try,vm-aot-mac-release-arm64-try,vm-aot-mac-release-x64-try,vm-aot-msan-linux-release-x64-try,vm-aot-obfuscate-linux-release-x64-try,vm-aot-optimization-level-linux-release-x64-try,vm-aot-tsan-linux-release-x64-try,vm-aot-ubsan-linux-release-x64-try,vm-aot-win-debug-arm64-try,vm-aot-win-debug-x64-try,vm-aot-win-debug-x64c-try,vm-appjit-linux-debug-x64-try,vm-asan-linux-release-arm64-try,vm-asan-linux-release-x64-try,vm-checked-mac-release-arm64-try,vm-ffi-android-debug-arm-try,vm-ffi-android-debug-arm64c-try,vm-ffi-qemu-linux-release-arm-try,vm-ffi-qemu-linux-release-riscv64-try,vm-fuchsia-release-arm64-try,vm-fuchsia-release-x64-try,vm-linux-debug-ia32-try,vm-linux-debug-x64-try,vm-linux-debug-x64c-try,vm-mac-debug-arm64-try,vm-mac-debug-x64-try,vm-msan-linux-release-arm64-try,vm-msan-linux-release-x64-try,vm-reload-linux-debug-x64-try,vm-reload-rollback-linux-debug-x64-try,vm-tsan-linux-release-arm64-try,vm-tsan-linux-release-x64-try,vm-ubsan-linux-release-arm64-try,vm-ubsan-linux-release-x64-try,vm-win-debug-arm64-try,vm-win-debug-x64-try,vm-win-debug-x64c-try,vm-win-release-ia32-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/407023
Commit-Queue: Daco Harkes <[email protected]>
Reviewed-by: Liam Appelbe <[email protected]>
Reviewed-by: Slava Egorov <[email protected]>

Author: dcharkes

runtime/vm/runtime_entry.cc

@@ -4413,7 +4413,9 @@ extern "C" Thread* DLRT_GetFfiCallbackMetadata(
     Isolate* current_isolate = nullptr;
     if (current_thread != nullptr) {
       current_isolate = current_thread->isolate();
-      ASSERT(current_thread->execution_state() == Thread::kThreadInNative);
+      if (current_thread->execution_state() != Thread::kThreadInNative) {
+        FATAL("Cannot invoke native callback from a leaf call.");
+      }
       current_thread->ExitSafepoint();
       current_thread->set_execution_state(Thread::kThreadInVM);
     }
@@ -4460,6 +4462,9 @@ extern "C" Thread* DLRT_GetFfiCallbackMetadata(
   if (current_thread->isolate() != target_isolate) {
     FATAL("Cannot invoke native callback from a different isolate.");
   }
+  if (current_thread->execution_state() != Thread::kThreadInNative) {
+    FATAL("Cannot invoke native callback from a leaf call.");
+  }
 
   // Set the execution state to VM while waiting for the safepoint to end.
   // This isn't strictly necessary but enables tests to check that we're not

tests/ffi/ffi.status

@@ -54,6 +54,7 @@ vmspecific_native_finalizer_isolate_groups_test: Skip # SpawnUri not available o
 [ $system == fuchsia ]
 async_void_function_callbacks_test/*: Skip # Test harness doesn't support multitest with Fuchsia
 ffi_induce_a_crash_test/*: Skip # Test harness doesn't support multitest with Fuchsia
+function_callbacks_leaf_test: SkipByDesign # Needs access to Dart executable
 function_callbacks_many_test/*: Skip # Test harness doesn't support multitest with Fuchsia
 function_callbacks_structs_by_value_generated_test/*: Skip # Test harness doesn't support multitest with Fuchsia
 function_callbacks_structs_by_value_native_callable_generated_test/*: Skip # Test harness doesn't support multitest with Fuchsia
@@ -71,6 +72,7 @@ vmspecific_pointer_load_il_test: SkipByDesign # Not bloating the Fuchsia test pa
 regress_47594_test: Skip # DynamicLibrary.process() is not available on Windows.
 
 [ $qemu ]
+function_callbacks_leaf_test: SkipByDesign # Needs access to Dart executable
 native_assets/*: SkipByDesign # Only intended to run on host oses with AOT binaries available, not available on QEMU.
 
 [ $simulator ]

tests/ffi/function_callbacks_leaf_test.dart

@@ -0,0 +1,60 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:ffi';
+import 'dart:io';
+
+import 'dylib_utils.dart';
+
+final ffiTestFunctions = dlopenPlatformSpecific('ffi_test_functions');
+
+typedef SimpleAdditionType = Void Function(Int32, Int32);
+
+void simpleAddition(int x, int y) {
+  print("simpleAddition($x, $y)");
+}
+
+void main(List<String> arguments) async {
+  if (arguments.isEmpty) {
+    for (int i in [1, 2, 3]) {
+      final result = Process.runSync(Platform.resolvedExecutable, [
+        Platform.script.toFilePath(),
+        '$i',
+      ]);
+      if (result.exitCode == 0) {
+        throw 'Expected non-0 exit code: ${result.exitCode}';
+      }
+      if (!result.stderr.contains(
+        'Cannot invoke native callback from a leaf call.',
+      )) {
+        throw "Expected stderr to contain 'Cannot invoke native callback from a leaf call.': ${result.stderr}";
+      }
+    }
+    return;
+  }
+  final Pointer<NativeFunction<SimpleAdditionType>> callbackPointer;
+  switch (arguments.single) {
+    case "1":
+      callbackPointer = Pointer.fromFunction<SimpleAdditionType>(
+        simpleAddition,
+      );
+    case "2":
+      callbackPointer =
+          NativeCallable<SimpleAdditionType>.isolateLocal(
+            simpleAddition,
+          ).nativeFunction;
+    case "3":
+      callbackPointer =
+          NativeCallable<SimpleAdditionType>.listener(
+            simpleAddition,
+          ).nativeFunction;
+    default:
+      throw "Unknown";
+  }
+  final function = callbackPointer.asFunction<void Function(int, int)>(
+    isLeaf: true,
+  );
+  function(3, 4);
+  print('We should have crashed by now.');
+}