Set assert handler from Zig

Signed-off-by: Maksym Pavlenko <pavlenko.maksym@gmail.com>

Author: mxpv

build.zig

@@ -170,6 +170,15 @@ pub fn build(b: *std.Build) !void {
             .optimize = optimize,
         });
 
+        // Add C wrapper
+        mod.addCSourceFile(.{
+            .file = b.path("src/handler.cpp"),
+            .flags = flags,
+        });
+        mod.addIncludePath(b.path("luau/VM/include"));
+        mod.addIncludePath(b.path("luau/Common/include"));
+        mod.addIncludePath(b.path("src"));
+
         mod.linkLibrary(luau_vm);
 
         // TODO: Make these optional
@@ -205,6 +214,15 @@ pub fn build(b: *std.Build) !void {
             }),
         });
 
+        // Add C wrapper
+        unit_tests.root_module.addCSourceFile(.{
+            .file = b.path("src/handler.cpp"),
+            .flags = flags,
+        });
+        unit_tests.root_module.addIncludePath(b.path("luau/VM/include"));
+        unit_tests.root_module.addIncludePath(b.path("luau/Common/include"));
+        unit_tests.root_module.addIncludePath(b.path("src"));
+
         // See https://zig.news/squeek502/code-coverage-for-zig-1dk1
         if (opts.cover) {
             unit_tests.setExecCmd(&[_]?[]const u8{

src/assert.zig

@@ -0,0 +1,19 @@
+const c = @cImport({
+    @cInclude("handler.h");
+});
+
+/// Assert handler function type for Luau VM assertions.
+pub const AssertHandler = c.Luau_AssertHandler;
+
+/// Set a custom assert handler for Luau VM assertions.
+///
+/// The handler is called when a Luau VM assertion fails, allowing custom error
+/// handling and debugging. The handler receives information about the failed assertion
+/// including expression, file, line number, and function name.
+///
+/// Parameters:
+/// - handler: Function pointer with signature (expr, file, line, func) -> c_int
+///   Returns 0 to abort, non-zero to continue execution
+pub inline fn setAssertHandler(handler: AssertHandler) void {
+    c.luau_set_assert_handler(handler);
+}

src/handler.cpp

@@ -0,0 +1,6 @@
+#include "handler.h"
+#include "Luau/Common.h"
+
+void luau_set_assert_handler(Luau_AssertHandler handler) {
+    Luau::assertHandler() = handler;
+}

src/handler.h

@@ -0,0 +1,13 @@
+#pragma once
+
+typedef int (*Luau_AssertHandler)(const char* expression, const char* file, int line, const char* function);
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void luau_set_assert_handler(Luau_AssertHandler handler);
+
+#ifdef __cplusplus
+}
+#endif

src/lua.zig

@@ -48,6 +48,7 @@ pub const Compiler = @import("compile.zig").Compiler;
 const userdata = @import("userdata.zig");
 const stack = @import("stack.zig");
 const alloc = @import("alloc.zig").alloc;
+const assert = @import("assert.zig");
 
 /// High-level Lua wrapper and main library entry point.
 /// Provides an idiomatic Zig interface with automatic type conversions for the Luau scripting language.
@@ -65,6 +66,9 @@ pub const Lua = struct {
         Compile,
     };
 
+    /// Assert handler function type for Luau VM assertions.
+    pub const AssertHandler = assert.AssertHandler;
+
     /// Initialize a new Lua state with optional custom allocator.
     ///
     /// Creates a new Luau virtual machine instance. Pass `null` to use Luau's built-in
@@ -939,6 +943,27 @@ pub const Lua = struct {
 
         return list.toOwnedSlice();
     }
+
+    /// Set a custom assert handler for Luau VM assertions.
+    ///
+    /// The assert handler is called when a Luau VM assertion fails, allowing custom error
+    /// handling and debugging. The handler receives information about the failed assertion
+    /// including expression, file, line number, and function name.
+    ///
+    /// Example:
+    /// ```zig
+    /// fn myAssertHandler(expr: [*c]const u8, file: [*c]const u8, line: c_int, func: [*c]const u8) callconv(.C) c_int {
+    ///     std.debug.print("Assertion failed: {s} at {s}:{} in {s}\n", .{expr, file, line, func});
+    ///     return 0; // Return 0 to abort
+    /// }
+    ///
+    /// Lua.setAssertHandler(myAssertHandler);
+    /// ```
+    ///
+    /// Note: The handler function must have C calling convention and return 0 to abort or non-zero to continue.
+    pub inline fn setAssertHandler(handler: AssertHandler) void {
+        assert.setAssertHandler(handler);
+    }
 };
 
 const expect = std.testing.expect;

src/tests.zig

@@ -524,3 +524,20 @@ test "checkArg userdata support" {
 
     try expectEq(lua.top(), 0);
 }
+
+fn testAssertHandler(expr: [*c]const u8, file: [*c]const u8, line: c_int, func: [*c]const u8) callconv(.C) c_int {
+    _ = expr;
+    _ = file;
+    _ = line;
+    _ = func;
+    return 1; // Continue execution
+}
+
+test "assert handler" {
+    const lua = try Lua.init(&std.testing.allocator);
+    defer lua.deinit();
+
+    // Verify the API works by setting and resetting the handler
+    Lua.setAssertHandler(testAssertHandler);
+    Lua.setAssertHandler(null);
+}