diff --git a/.gitmodules b/.gitmodules index 38d012bc..a34ce3f8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "third-party/luajit20"] path = third-party/luajit20 url = https://luajit.org/git/luajit.git +[submodule "third-party/luau"] + path = third-party/luau + url = https://github.com/luau-lang/luau diff --git a/Makefile b/Makefile index 3ea6b067..1374f493 100644 --- a/Makefile +++ b/Makefile @@ -26,11 +26,16 @@ all: local local: LUPA_WITH_LUA_DLOPEN=$(WITH_LUA_DLOPEN) ${PYTHON} setup.py build_ext --inplace $(WITH_PARALLEL) $(WITH_CYTHON) +local_luau: + LUPA_WITH_LUA_DLOPEN=$(WITH_LUA_DLOPEN) ${PYTHON} setup.py --use-luau build_ext --inplace $(WITH_PARALLEL) $(WITH_CYTHON) + sdist dist/lupa-$(VERSION).tar.gz: ${PYTHON} setup.py sdist test: local PYTHONPATH=. $(PYTHON) -m unittest -v lupa.tests.suite +test_luau: local_luau + PYTHONPATH=. $(PYTHON) -m unittest -v lupa.tests.suite clean: rm -fr build lupa/_lupa*.so lupa/lua*.pyx lupa/*.c diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 680a9cf6..11880ba5 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -75,9 +75,9 @@ include "lock.pxi" cdef int _LUA_VERSION = lua.read_lua_version(NULL) +cdef int LUAU = lua.is_luau() LUA_VERSION = (_LUA_VERSION // 100, _LUA_VERSION % 100) - if lua.LUA_MAXINTEGER > 0: LUA_MININTEGER, LUA_MAXINTEGER = (lua.LUA_MININTEGER, lua.LUA_MAXINTEGER) elif sizeof(lua.lua_Integer) >= sizeof(long long): # probably not larger @@ -246,6 +246,7 @@ cdef class LuaRuntime: cdef object _attribute_setter cdef bint _unpack_returned_tuples cdef MemoryStatus _memory_status + cdef bint _luau_running_gc # It is not safe to run Luau VM operations if GC is running def __cinit__(self, encoding='UTF-8', source_encoding=None, attribute_filter=None, attribute_handlers=None, @@ -254,6 +255,12 @@ cdef class LuaRuntime: max_memory=None): cdef lua_State* L + # TODO: Use pcall for globals as well on Luau if needed + # + # Also, remove this once Luau support is more stable + if LUAU and max_memory and max_memory < 100000: + raise LuaError("Luau needs a max_memory of at least 100000 for now!") + if max_memory is None: L = lua.luaL_newstate() self._memory_status.limit = -1 @@ -271,6 +278,7 @@ cdef class LuaRuntime: raise ValueError("attribute_filter must be callable") self._attribute_filter = attribute_filter self._unpack_returned_tuples = unpack_returned_tuples + self._luau_running_gc = False if attribute_handlers: raise_error = False @@ -288,7 +296,10 @@ cdef class LuaRuntime: raise ValueError("attribute_filter and attribute_handlers are mutually exclusive") self._attribute_getter, self._attribute_setter = getter, setter - lua.lua_atpanic(L, &_lua_panic) + if LUAU: + lua.lua_setpanicfunc(L, &_lua_panic_luau) + else: + lua.lua_atpanic(L, &_lua_panic) lua.luaL_openlibs(L) self.init_python_lib(register_eval, register_builtins) @@ -399,6 +410,16 @@ cdef class LuaRuntime: raise return 0 + @cython.final + cdef int store_raised_exception_luaugc(self, bytes lua_error_msg) except -1: + # Special variant of store_raised_exception() for Luau's + # GC metamethod + try: + self._raised_exception = tuple(exc_info()) + except: + raise + return 0 + @cython.final cdef bytes _source_encode(self, string): if isinstance(string, unicode): @@ -633,12 +654,15 @@ cdef class LuaRuntime: # create 'python' lib luaL_openlib(L, "python", py_lib, 0) # lib lua.lua_pushlightuserdata(L, self) # lib udata - lua.lua_pushcclosure(L, py_args, 1) # lib function + lua.lua_pushcclosured(L, py_args, 1) # lib function lua.lua_setfield(L, -2, "args") # lib # register our own object metatable lua.luaL_newmetatable(L, POBJECT) # lib metatbl - luaL_openlib(L, NULL, py_object_lib, 0) + if LUAU: + luaL_openlib(L, NULL, py_object_lib_luau, 0) + else: + luaL_openlib(L, NULL, py_object_lib, 0) lua.lua_pop(L, 1) # lib # create and store the python references table @@ -764,7 +788,7 @@ cdef Py_ssize_t get_object_length(LuaRuntime runtime, lua_State* L, int index) e cdef size_t length check_lua_stack(L, 1) lua.lua_pushvalue(L, index) # value - lua.lua_pushcclosure(L, get_object_length_from_lua, 1) # closure + lua.lua_pushcclosured(L, get_object_length_from_lua, 1) # closure result = lua.lua_pcall(L, 0, 1, 0) if result: # err raise_lua_error(runtime, L, result) # @@ -886,7 +910,7 @@ cdef class _LuaObject: locked = lock_runtime(runtime, blocking=False) if locked: lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, ref) - runtime.clean_up_pending_unrefs() # just in case + runtime.clean_up_pending_unrefs() # for non-luau, just in case, for luau, needed unlock_runtime(runtime) return runtime.add_pending_unref(ref) @@ -910,7 +934,7 @@ cdef class _LuaObject: def __call__(self, *args): assert self._runtime is not None - cdef lua_State* L = self._state + cdef lua_State* L = self._state if not LUAU else self._runtime._state if not lock_runtime(self._runtime): raise RuntimeError("failed to acquire thread lock") try: @@ -1011,7 +1035,7 @@ cdef class _LuaObject: old_top = lua.lua_gettop(L) try: check_lua_stack(L, 3) - lua.lua_pushcfunction(L, get_from_lua_table) # func + lua.lua_pushcfunctiond(L, get_from_lua_table) # func self.push_lua_object(L) # func obj lua_type = lua.lua_type(L, -1) if lua_type == lua.LUA_TFUNCTION or lua_type == lua.LUA_TTHREAD: @@ -1228,6 +1252,11 @@ cdef class _LuaThread(_LuaObject): self._arguments = None return resume_lua_thread(self, args) + def __call__(self, *args): + if LUAU: + raise TypeError("Lua threads cannot be called directly on Luau (lua_pcall on threads is not supported)") + super().__call__(*args) + def send(self, value): """Send a value into the coroutine. If the value is a tuple, send the unpacked elements. @@ -1358,7 +1387,7 @@ cdef class _LuaIter: locked = lock_runtime(runtime, blocking=False) if locked: lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, ref) - runtime.clean_up_pending_unrefs() # just in case + runtime.clean_up_pending_unrefs() # for non luau, just in case, for luau, needed unlock_runtime(runtime) return runtime.add_pending_unref(ref) @@ -1673,7 +1702,10 @@ cdef bint py_to_lua_custom(LuaRuntime runtime, lua_State *L, object o, int type_ lua.lua_pop(L, 1) # tbl # create new wrapper for Python object - py_obj = lua.lua_newuserdata(L, sizeof(py_object)) + if LUAU: + py_obj = lua.lua_newuserdatadtor(L, sizeof(py_object), py_object_gc_luau) + else: + py_obj = lua.lua_newuserdata(L, sizeof(py_object)) py_obj.obj = o # tbl udata py_obj.runtime = runtime py_obj.type_flags = type_flags @@ -1884,6 +1916,9 @@ cdef call_lua(LuaRuntime runtime, lua_State *L, tuple args): return execute_lua_call(runtime, L, len(args)) cdef object execute_lua_call(LuaRuntime runtime, lua_State *L, Py_ssize_t nargs): + if LUAU and lua.lua_status(L) != 0: + # Luau does not support calling functions in yielding threads + raise LuaError("cannot call function in non-suspended Luau thread") cdef int result_status cdef object result # call into Lua @@ -1903,7 +1938,7 @@ cdef object execute_lua_call(LuaRuntime runtime, lua_State *L, Py_ssize_t nargs) result_status = lua.lua_pcall(L, nargs, lua.LUA_MULTRET, has_lua_traceback_func) if has_lua_traceback_func: lua.lua_remove(L, 1) - runtime.clean_up_pending_unrefs() + runtime.clean_up_pending_unrefs() # for luau, this acts as a safe place to clear out GC'd refs results = unpack_lua_results(runtime, L) if result_status: if isinstance(results, BaseException): @@ -2001,6 +2036,18 @@ cdef int _lua_panic(lua_State *L) noexcept nogil: fflush(stderr) return 0 # return to Lua to abort +cdef void _lua_panic_luau(lua_State *L, int errcode) noexcept nogil: + cdef const char* msg = lua.lua_tostring(L, -1) + cdef const char* msg_typename = NULL + if msg == NULL: + msg_typename = lua.luaL_typename(L, -1) + if msg_typename != NULL: + msg = msg_typename + msg = "error object is not a string" + cdef char* message = "PANIC: unprotected error in call to Lua API (%s) with err code (%d)\n" + fprintf(stderr, message, msg, errcode) + fflush(stderr) + return # return to Lua to abort ################################################################################ # Python support in Lua @@ -2060,9 +2107,37 @@ cdef int py_object_gc(lua_State* L) noexcept nogil: py_obj = unpack_userdata(L, 1) if py_obj is not NULL and py_obj.obj is not NULL: if py_object_gc_with_gil(py_obj, L): - return lua.lua_error(L) # never returns! + return lua.lua_errord(L) # never returns! return 0 +cdef void py_object_gc_luau(void *ud) noexcept with gil: + # Note: Luau uses its own GC mechanism that does not use the __gc metamethod. + # This GC mechanism is special as it does not allow for performing any Luau operations + cdef _PyReference pyref + cdef py_object* py_obj = ud + runtime = py_obj.runtime + runtime._luau_running_gc = True + try: + refkey = build_pyref_key(py_obj.obj, py_obj.type_flags) + pyref = <_PyReference>runtime._pyrefs_in_lua.pop(refkey) + except (TypeError, KeyError): + runtime._luau_running_gc = False + return # runtime was already cleared during GC, nothing left to do + except: + try: runtime.store_raised_exception_luaugc(b'error while cleaning up a Python object') + finally: + runtime._luau_running_gc = False + return + else: + # Luau GC does not support any VM operations within GC + # + # So, we need to delay the unref operation to a safe point + runtime.add_pending_unref(pyref._ref) + finally: + py_obj.obj = NULL + runtime._luau_running_gc = False + + # calling Python objects cdef bint call_python(LuaRuntime runtime, lua_State *L, py_object* py_obj) except -1: @@ -2127,7 +2202,7 @@ cdef int py_object_call(lua_State* L) noexcept nogil: cdef py_object* py_obj = unpack_python_argument_or_jump(L, 1) # may not return on error! result = py_call_with_gil(L, py_obj) if result < 0: - return lua.lua_error(L) # never returns! + return lua.lua_errord(L) # never returns! return result # str() support for Python objects @@ -2154,7 +2229,7 @@ cdef int py_object_str(lua_State* L) noexcept nogil: cdef py_object* py_obj = unpack_python_argument_or_jump(L, 1) # may not return on error! result = py_str_with_gil(L, py_obj) if result < 0: - return lua.lua_error(L) # never returns! + return lua.lua_errord(L) # never returns! return result # item access for Python objects @@ -2226,7 +2301,7 @@ cdef int py_object_getindex(lua_State* L) noexcept nogil: cdef py_object* py_obj = unpack_python_argument_or_jump(L, 1) # may not return on error! result = py_object_getindex_with_gil(L, py_obj) if result < 0: - return lua.lua_error(L) # never returns! + return lua.lua_errord(L) # never returns! return result @@ -2246,7 +2321,7 @@ cdef int py_object_setindex(lua_State* L) noexcept nogil: cdef py_object* py_obj = unpack_python_argument_or_jump(L, 1) # may not return on error! result = py_object_setindex_with_gil(L, py_obj) if result < 0: - return lua.lua_error(L) # never returns! + return lua.lua_errord(L) # never returns! return result # special methods for Lua wrapped Python objects @@ -2260,11 +2335,19 @@ cdef lua.luaL_Reg *py_object_lib = [ lua.luaL_Reg(name = NULL, func = NULL), ] +cdef lua.luaL_Reg *py_object_lib_luau = [ + lua.luaL_Reg(name = "__call", func = py_object_call), + lua.luaL_Reg(name = "__index", func = py_object_getindex), + lua.luaL_Reg(name = "__newindex", func = py_object_setindex), + lua.luaL_Reg(name = "__tostring", func = py_object_str), + lua.luaL_Reg(name = NULL, func = NULL), +] + ## # Python helper functions for Lua cdef inline py_object* unpack_single_python_argument_or_jump(lua_State* L) noexcept nogil: if lua.lua_gettop(L) > 1: - lua.luaL_argerror(L, 2, "invalid arguments") # never returns! + lua.luaL_argerrord(L, 2, "invalid arguments") # never returns! return unpack_python_argument_or_jump(L, 1) cdef inline py_object* unpack_python_argument_or_jump(lua_State* L, int n) noexcept nogil: @@ -2276,9 +2359,9 @@ cdef inline py_object* unpack_python_argument_or_jump(lua_State* L, int n) noexc py_obj = unpack_wrapped_pyfunction(L, n) if not py_obj: - lua.luaL_argerror(L, n, "not a python object") # never returns! + lua.luaL_argerrord(L, n, "not a python object") # never returns! if not py_obj.obj: - lua.luaL_argerror(L, n, "deleted python object") # never returns! + lua.luaL_argerrord(L, n, "deleted python object") # never returns! return py_obj @@ -2295,7 +2378,7 @@ cdef int py_wrap_object_protocol(lua_State* L, int type_flags) noexcept nogil: cdef py_object* py_obj = unpack_single_python_argument_or_jump(L) # never returns on error! result = py_wrap_object_protocol_with_gil(L, py_obj, type_flags) if result < 0: - return lua.lua_error(L) # never returns! + return lua.lua_errord(L) # never returns! return result cdef int py_as_attrgetter(lua_State* L) noexcept nogil: @@ -2306,7 +2389,7 @@ cdef int py_as_itemgetter(lua_State* L) noexcept nogil: cdef int py_as_function(lua_State* L) noexcept nogil: cdef py_object* py_obj = unpack_single_python_argument_or_jump(L) # never returns on error! - lua.lua_pushcclosure(L, py_asfunc_call, 1) + lua.lua_pushcclosured(L, py_asfunc_call, 1) return 1 # iteration support for Python objects in Lua @@ -2315,14 +2398,14 @@ cdef int py_iter(lua_State* L) noexcept nogil: cdef py_object* py_obj = unpack_single_python_argument_or_jump(L) # never returns on error! result = py_iter_with_gil(L, py_obj, 0) if result < 0: - return lua.lua_error(L) # never returns! + return lua.lua_errord(L) # never returns! return result cdef int py_iterex(lua_State* L) noexcept nogil: cdef py_object* py_obj = unpack_single_python_argument_or_jump(L) # never returns on error! result = py_iter_with_gil(L, py_obj, OBJ_UNPACK_TUPLE) if result < 0: - return lua.lua_error(L) # never returns! + return lua.lua_errord(L) # never returns! return result cdef int convert_to_lua_Integer(lua_State* L, int idx, lua.lua_Integer* integer) noexcept nogil: @@ -2338,17 +2421,17 @@ cdef int convert_to_lua_Integer(lua_State* L, int idx, lua.lua_Integer* integer) cdef int py_enumerate(lua_State* L) noexcept nogil: if lua.lua_gettop(L) > 2: - lua.luaL_argerror(L, 3, "invalid arguments") # never returns! + lua.luaL_argerrord(L, 3, "invalid arguments") # never returns! cdef py_object* py_obj = unpack_python_argument_or_jump(L, 1) cdef lua.lua_Integer start if lua.lua_gettop(L) == 2: if convert_to_lua_Integer(L, -1, &start) < 0: - return lua.lua_error(L) # never returns + return lua.lua_errord(L) # never returns else: start = 0 result = py_enumerate_with_gil(L, py_obj, start) if result < 0: - return lua.lua_error(L) # never returns! + return lua.lua_errord(L) # never returns! return result @@ -2383,7 +2466,7 @@ cdef int py_push_iterator(LuaRuntime runtime, lua_State* L, iterator, int type_f Returns the number of pushed values """ # push the iterator function - lua.lua_pushcfunction(L, py_iter_next) + lua.lua_pushcfunctiond(L, py_iter_next) # push the wrapped iterator object as for-loop state object if runtime._unpack_returned_tuples: type_flags |= OBJ_UNPACK_TUPLE @@ -2400,7 +2483,7 @@ cdef int py_iter_next(lua_State* L) noexcept nogil: cdef py_object* py_obj = unpack_python_argument_or_jump(L, 1) # may not return on error! result = py_iter_next_with_gil(L, py_obj) if result < 0: - return lua.lua_error(L) # never returns! + return lua.lua_errord(L) # never returns! return result cdef int py_iter_next_with_gil(lua_State* L, py_object* py_iter) noexcept with gil: @@ -2456,11 +2539,11 @@ cdef int py_args(lua_State* L) noexcept nogil: cdef PyObject* runtime runtime = lua.lua_touserdata(L, lua.lua_upvalueindex(1)) if not runtime: - return lua.luaL_error(L, "missing runtime") + return lua.luaL_errord(L, "missing runtime") lua.luaL_checktype(L, 1, lua.LUA_TTABLE) result = py_args_with_gil(runtime, L) if result < 0: - return lua.lua_error(L) # never returns! + return lua.lua_errord(L) # never returns! return result # overflow handler setter @@ -2469,7 +2552,7 @@ cdef int py_set_overflow_handler(lua_State* L) noexcept nogil: if (not lua.lua_isnil(L, 1) and not lua.lua_isfunction(L, 1) and not unpack_python_argument_or_jump(L, 1)): - return lua.luaL_argerror(L, 1, "expected nil, a Lua function or a callable Python object") + return lua.luaL_argerrord(L, 1, "expected nil, a Lua function or a callable Python object") # hdl [...] lua.lua_settop(L, 1) # hdl lua.lua_setfield(L, lua.LUA_REGISTRYINDEX, LUPAOFH) # @@ -2496,7 +2579,7 @@ cdef void luaL_setfuncs(lua_State *L, const lua.luaL_Reg *l, int nup) noexcept: while l.name != NULL: for i in range(nup): lua.lua_pushvalue(L, -nup) - lua.lua_pushcclosure(L, l.func, nup) + lua.lua_pushcclosured(L, l.func, nup) lua.lua_setfield(L, -(nup + 2), l.name) l += 1 lua.lua_pop(L, nup) @@ -2547,7 +2630,7 @@ cdef void luaL_pushmodule(lua_State *L, const char *modname, int size_hint) noex lua.lua_pop(L, 1) lua.lua_getglobal(L, '_G') if luaL_findtable(L, 0, modname, size_hint) != NULL: - lua.luaL_error(L, "name conflict for module '%s'", modname) + lua.luaL_errord(L, "name conflict for module '%s'", modname) lua.lua_pushvalue(L, -1) lua.lua_setfield(L, -3, modname) lua.lua_remove(L, -2) diff --git a/lupa/luaapi.pxd b/lupa/luaapi.pxd index 03aad5a1..6da90347 100644 --- a/lupa/luaapi.pxd +++ b/lupa/luaapi.pxd @@ -75,6 +75,15 @@ cdef extern from "lua.h" nogil: const lua_Number *lua_version(lua_State *L) lua_CFunction lua_atpanic (lua_State *L, lua_CFunction panicf) + #ifdef USE_LUAU + void lua_setpanicfunc (lua_State *L, void (*panic)(lua_State* L, int errcode)) + #endif + + #ifdef USE_LUAU + void* lua_newuserdatadtor (lua_State* L, size_t sz, void (*dtor)(void*)) + #else + #define lua_newuserdatadtor(L, sz, dtor) luaL_error(L, "lua_newuserdatadtor is not available") + #endif # basic stack manipulation int lua_gettop (lua_State *L) @@ -117,7 +126,11 @@ cdef extern from "lua.h" nogil: void lua_pushstring (lua_State *L, char *s) char *lua_pushvfstring (lua_State *L, char *fmt, va_list argp) char *lua_pushfstring (lua_State *L, char *fmt, ...) - void lua_pushcclosure (lua_State *L, lua_CFunction fn, int n) + #ifdef USE_LUAU + void lua_pushcclosured (lua_State *L, lua_CFunction fn, int n) + #else + void lua_pushcclosure (lua_State *L, lua_CFunction fn, int n) # Will be used to define lua_pushcclosured + #endif void lua_pushboolean (lua_State *L, bint b) void lua_pushlightuserdata (lua_State *L, void *p) int lua_pushthread (lua_State *L) @@ -168,7 +181,11 @@ cdef extern from "lua.h" nogil: int lua_gc (lua_State *L, int what, int data) # miscellaneous functions + #ifdef USE_LUAU + int lua_errord (lua_State *L) + #else int lua_error (lua_State *L) + #endif int lua_next (lua_State *L, int idx) void lua_concat (lua_State *L, int n) lua_Alloc lua_getallocf (lua_State *L, void **ud) @@ -181,7 +198,11 @@ cdef extern from "lua.h" nogil: void lua_pop(lua_State *L, int n) # lua_settop(L, -(n)-1) void lua_newtable(lua_State *L) # lua_createtable(L, 0, 0) void lua_register(lua_State *L, char* n, lua_CFunction f) # (lua_pushcfunction(L, (f)), lua_setglobal(L, (n))) + #ifdef USE_LUAU + void lua_pushcfunctiond(lua_State *L, lua_CFunction fn) + #else void lua_pushcfunction(lua_State *L, lua_CFunction fn) # lua_pushcclosure(L, (f), 0) + #endif size_t lua_strlen(lua_State *L, int i) # lua_objlen(L, (i)) bint lua_isfunction(lua_State *L, int n) # (lua_type(L, (n)) == LUA_TFUNCTION) @@ -292,7 +313,12 @@ cdef extern from "lauxlib.h" nogil: int luaL_getmetafield (lua_State *L, int obj, char *e) int luaL_callmeta (lua_State *L, int obj, char *e) int luaL_typerror (lua_State *L, int narg, char *tname) + + #ifdef USE_LUAU + int luaL_argerrord (lua_State *L, int numarg, char *extramsg) + #else int luaL_argerror (lua_State *L, int numarg, char *extramsg) + #endif char *luaL_checklstring (lua_State *L, int numArg, size_t *l) char *luaL_optlstring (lua_State *L, int numArg, char *default, size_t *l) lua_Number luaL_checknumber (lua_State *L, int numArg) @@ -309,7 +335,12 @@ cdef extern from "lauxlib.h" nogil: void *luaL_checkudata (lua_State *L, int ud, char *tname) void luaL_where (lua_State *L, int lvl) + + #ifdef USE_LUAU + int luaL_errord (lua_State *L, char *fmt, ...) + #else int luaL_error (lua_State *L, char *fmt, ...) + #endif int luaL_checkoption (lua_State *L, int narg, char *default, char *lst[]) @@ -423,7 +454,6 @@ cdef extern from "lualib.h": void luaL_openlibs(lua_State *L) - cdef extern from * nogil: # Compatibility definitions for Lupa. """ @@ -431,7 +461,7 @@ cdef extern from * nogil: #define __lupa_lua_resume lua_resume #else LUA_API int __lupa_lua_resume (lua_State *L, lua_State *from, int nargs, int* nresults) { - #if LUA_VERSION_NUM >= 502 + #if LUA_VERSION_NUM >= 502 || USE_LUAU int status = lua_resume(L, from, nargs); #else int status = lua_resume(L, nargs); @@ -441,6 +471,14 @@ cdef extern from * nogil: } #endif + #ifndef USE_LUAU + #define lua_pushcclosured lua_pushcclosure + #define lua_pushcfunctiond lua_pushcfunction + #define luaL_errord luaL_error + #define lua_errord lua_error + #define luaL_argerrord luaL_argerror + #endif + #if LUA_VERSION_NUM >= 502 #define lua_objlen(L, i) lua_rawlen(L, (i)) #endif @@ -449,8 +487,11 @@ cdef extern from * nogil: #define lua_isinteger(L, i) (((void) i), 0) #endif - #if LUA_VERSION_NUM < 502 + #if LUA_VERSION_NUM < 502 && !USE_LUAU #define lua_tointegerx(L, i, isnum) (*(isnum) = lua_isnumber(L, i), lua_tointeger(L, i)) + #endif + + #if LUA_VERSION_NUM < 502 #define luaL_loadbufferx(L, buff, sz, name, mode) (((void)mode), luaL_loadbuffer(L, buff, sz, name)) #endif @@ -463,12 +504,18 @@ cdef extern from * nogil: #else #error Lupa requires at least Lua 5.1 or LuaJIT 2.x #endif + #ifdef USE_LUAU + #define is_luau() ((int) 1) + #else + #define is_luau() ((int) 0) + #endif #if LUA_VERSION_NUM < 502 #define lua_pushglobaltable(L) lua_pushvalue(L, LUA_GLOBALSINDEX) #endif """ int read_lua_version(lua_State *L) + int is_luau() int lua_isinteger(lua_State *L, int idx) lua_Integer lua_tointegerx (lua_State *L, int idx, int *isnum) void lua_pushglobaltable (lua_State *L) diff --git a/setup.py b/setup.py index ce3357dd..93e57fb4 100644 --- a/setup.py +++ b/setup.py @@ -214,6 +214,237 @@ def no_lua_error(): print(error) return {} +def use_bundled_luau(path, macros): + import copy + libname = os.path.basename(path.rstrip(os.sep)) + assert 'luau' in libname, libname + print('Using bundled luau in %s' % libname) + print('Building Luau for %r in %s' % (platform, libname)) + + build_env = dict(os.environ) + src_dir = path + + if not os.path.exists(os.path.join(src_dir, "build")): + os.mkdir(os.path.join(src_dir, "build")) + + base_cflags = [ + "-fPIC", "-O0", "-DLUA_USE_LONGJMP=1", "-DLUA_VECTOR_SIZE=3", "-fno-math-errno", "-DLUAI_MAXCSTACK=8000", "-DLUA_API=extern \"C\"", "-DLUACODEGEN_API=extern \"C\"", "-DLUACODE_API=extern \"C\"", + "-fexceptions" + ] + base_cxxflags = base_cflags + ["-std=c++17"] + + for lib in [ + ("Ast", "luauast"), + ("CodeGen", "luaucodegen"), + ("Compiler", "luaucompiler"), + ("VM", "luauvm"), + ]: + if not os.path.exists(os.path.join(src_dir, "build", lib[1])): + os.mkdir(os.path.join(src_dir, "build", lib[1])) + + # Skip if the library is already built + static_lib_path = os.path.join(src_dir, "build", "lib" + lib[1] + ".a") + if os.path.exists(static_lib_path): + print("Static library %s already exists, skipping build." % static_lib_path) + continue + + lib_src_dir = os.path.join(src_dir, lib[0], "src") + lib_include_dir = os.path.join(src_dir, lib[0], "include") + + lib_build_env = { + 'CFLAGS': copy.copy(base_cflags), + 'CXXFLAGS': copy.copy(base_cxxflags) + } + + common_includes = [ + os.path.join(src_dir, "Common", "include"), + ] + + if lib[0] == "CodeGen": # Codegen needs VM includes (src of VM also includes headers needed by Codegen) + common_includes.append(os.path.join(src_dir, "VM", "include")) + common_includes.append(os.path.join(src_dir, "VM", "src")) + elif lib[0] == "Compiler": # Compiler needs Ast includes + common_includes.append(os.path.join(src_dir, "Ast", "include")) + + lib_build_env["CFLAGS"].append("-I" + lib_include_dir) + lib_build_env["CFLAGS"].extend("-I" + inc for inc in common_includes) + lib_build_env["CXXFLAGS"].append("-I" + lib_include_dir) + lib_build_env["CXXFLAGS"].extend("-I" + inc for inc in common_includes) + + # Find all cpp files in the src directory + cpp_files = [] + for root, dirs, files in os.walk(lib_src_dir): + for file in files: + if file.endswith(".cpp"): + cpp_files.append(os.path.join(root, file)) + if not cpp_files: + raise RuntimeError("No .cpp files found in " + lib_src_dir) + + # Compile the library + object_files = [] + for cpp_file in cpp_files: + obj_file = os.path.splitext(os.path.basename(cpp_file))[0] + '.o' + obj_file_path = os.path.join(src_dir, "build", lib[1], obj_file) + compile_command = ["g++", "-c", cpp_file, "-o", obj_file_path] + lib_build_env['CXXFLAGS'] + print("Compiling %s" % cpp_file) + output = subprocess.check_output(compile_command, cwd=lib_src_dir) + object_files.append(obj_file_path) + + # Create the static library + static_lib_path = os.path.join(src_dir, "build", "lib" + lib[1] + ".a") + ar_command = ["ar", "rcs", static_lib_path] + object_files + print("Creating static library %s" % static_lib_path) + output = subprocess.check_output(ar_command, cwd=lib_src_dir) + if b'error' in output.lower(): + print("Creating static library Luau did not report success:") + print(output.decode().strip()) + print("## Creating static library Luau may have failed ##") + + # This is a bit of a hack, but luau doesnt have a lauxlib.h, so we make a new lauxlib.h + # that merely includes lualib.h + lauxlib_h_path = os.path.join(src_dir, "build", "lauxlib.h") + if not os.path.exists(lauxlib_h_path): + with open(lauxlib_h_path, 'w', encoding='us-ascii') as f: + f.write(""" +#pragma once +#include "lualib.h" +#include "luacode.h" +#include +#define USE_LUAU 1 +#define LUA_VERSION_NUM 501 + +#define ref_freelist 0 + +// Polyfill for luaL_ref +// From https://github.com/lua/lua/blob/v5-2/lauxlib.c +LUALIB_API int luaL_ref (lua_State *L, int t) { + int ref; + if (lua_isnil(L, -1)) { + lua_pop(L, 1); /* remove from stack */ + return LUA_REFNIL; /* `nil' has a unique fixed reference */ + } + t = lua_absindex(L, t); + lua_rawgeti(L, t, ref_freelist); /* get first free element */ + ref = (int)lua_tointeger(L, -1); /* ref = t[ref_freelist] */ + lua_pop(L, 1); /* remove it from stack */ + if (ref != 0) { /* any free element? */ + lua_rawgeti(L, t, ref); /* remove it from list */ + lua_rawseti(L, t, ref_freelist); /* (t[ref_freelist] = t[ref]) */ + } + else /* no free elements */ + ref = (int)lua_objlen(L, t) + 1; /* get a new reference */ + lua_rawseti(L, t, ref); + return ref; +} + +// Polyfill for luaL_unref +// From https://github.com/lua/lua/blob/v5-2/lauxlib.c +LUALIB_API void luaL_unref (lua_State *L, int t, int ref) { + if (ref >= 0) { + t = lua_absindex(L, t); + lua_rawgeti(L, t, ref_freelist); + lua_rawseti(L, t, ref); /* t[ref] = t[ref_freelist] */ + lua_pushinteger(L, ref); + lua_rawseti(L, t, ref_freelist); /* t[ref_freelist] = ref */ + } +} + +// Define lua_pushcclosured using lua_pushcclosurek +LUALIB_API void lua_pushcclosured (lua_State *L, lua_CFunction fn, int n) { + lua_pushcclosurek(L, fn, NULL, n, NULL); +} + +// Define lua_pushcfunctiond using lua_pushcfunction +LUALIB_API void lua_pushcfunctiond (lua_State *L, lua_CFunction fn) { + lua_pushcfunction(L, fn, NULL); +} + +// Dummy implementation of lua_atpanic +LUALIB_API lua_CFunction lua_atpanic (lua_State *L, lua_CFunction panicf) { + luaL_error(L, "lua_atpanic is not supported in Luau. Use lua_setpanicfunc instead."); + return NULL; // never reached +} + +LUALIB_API void lua_setpanicfunc (lua_State *L, void (*panic)(lua_State* L, int errcode)) { + lua_callbacks(L)->panic = panic; +} + +// Define luaL_errord as luaL_error with return of int +LUALIB_API int luaL_errord (lua_State *L, const char *fmt, ...) { + va_list argp; + va_start(argp, fmt); + luaL_error(L, fmt, argp); + va_end(argp); + return 0; // never reached +} + +// Define lua_errord as lua_error with return of int +LUALIB_API int lua_errord (lua_State *L) { + lua_error(L); + return 0; // never reached +} + +// Define luaL_argerrord as luaL_argerror with return of int +LUALIB_API int luaL_argerrord (lua_State *L, int arg, const char *extramsg) { + luaL_argerror(L, arg, extramsg); + return 0; // never reached +} + +// Polyfill for lua_getstack via lua_getinfo on Luau +// May not be fully correct but should be the Luau equivalent +#define lua_getstack(L, level, ar) lua_getinfo(L, level, "", ar) + +// Compile with luau_compile +// TODO: Compile using lupa's memory allocator +// and support luau env parameter +void chunk_dtor(void *ud) { + if(ud == NULL) { + return; + } + char* data_to_free = *(char**)ud; + if(data_to_free == NULL) { + return; + } + free(data_to_free); // This will always be called even on error etc. +} + +// Polyfill for luaL_loadbuffer +// Luau doesnt provide either luaL_loadbuffer or luaL_loadbufferx etc. +LUALIB_API int luaL_loadbuffer (lua_State *L, const char *buffer, size_t size, const char *name) { + bool textChunk = (size == 0 || buffer[0] >= '\t'); + if (textChunk) { + void* ud = lua_newuserdatadtor(L, sizeof(char*), chunk_dtor); + size_t outsize = 0; + char* data = luau_compile(buffer, size, NULL, &outsize); + // ptr::write(data_ud, data); + *(char**)ud = data; // Now, the dtor will always free this + int status = luau_load(L, name, data, outsize, 0); // TODO: Support env parameter for optimized chunk environment loading + lua_replace(L, -2); // Replace the userdata with the result + return status; + } else { + // Binary chunk, load with luau_load + return luau_load(L, name, buffer, size, 0); // TODO: + } +} +""") + + return { + 'include_dirs': [ + os.path.join(src_dir, "Common", "include"), + os.path.join(src_dir, "Ast", "include"), + os.path.join(src_dir, "CodeGen", "include"), + os.path.join(src_dir, "Compiler", "include"), + os.path.join(src_dir, "VM", "include"), + os.path.join(src_dir, "VM", "src"), + os.path.join(src_dir, "build"), + ], + 'extra_objects': [ + os.path.join(src_dir, "build", "libluaucompiler.a"), + os.path.join(src_dir, "build", "libluaucodegen.a"), + os.path.join(src_dir, "build", "libluauast.a"), + os.path.join(src_dir, "build", "libluauvm.a")], + 'libversion': libname, + } def use_bundled_luajit(path, macros): libname = os.path.basename(path.rstrip(os.sep)) @@ -253,6 +484,8 @@ def use_bundled_lua(path, macros): libname = os.path.basename(path.rstrip(os.sep)) if 'luajit' in libname: return use_bundled_luajit(path, macros) + elif 'luau' in libname: + return use_bundled_luau(path, macros) print('Using bundled Lua in %s' % libname) @@ -349,22 +582,32 @@ def has_option(name): option_no_bundle = has_option('--no-bundle') option_use_bundle = has_option('--use-bundle') option_no_luajit = has_option('--no-luajit') +option_use_luau = has_option('--use-luau') +if option_use_luau and option_no_bundle: + print("Cannot use --use-luau together with --no-bundle") + sys.exit(1) configs = get_lua_build_from_arguments() if not configs and not option_no_bundle: - configs = [ - use_bundled_lua(lua_bundle_path, c_defines) - for lua_bundle_path in glob.glob(os.path.join(basedir, 'third-party', 'lua*' + os.sep)) - if not ( - False - # LuaJIT 2.0 on macOS requires a CPython linked with "-pagezero_size 10000 -image_base 100000000" - # http://t-p-j.blogspot.com/2010/11/lupa-on-os-x-with-macports-python-26.html - # LuaJIT 2.1-alpha3 fails at runtime. - or (platform == 'darwin' and 'luajit' in os.path.basename(lua_bundle_path.rstrip(os.sep))) - # Let's restrict LuaJIT to x86_64 for now. - or (get_machine() not in ("x86_64", "AMD64") and 'luajit' in os.path.basename(lua_bundle_path.rstrip(os.sep))) - ) - ] + if option_use_luau: + configs = [ + use_bundled_luau(lua_bundle_path, c_defines) + for lua_bundle_path in glob.glob(os.path.join(basedir, 'third-party', 'luau*' + os.sep)) + ] + else: + configs = [ + use_bundled_lua(lua_bundle_path, c_defines) + for lua_bundle_path in glob.glob(os.path.join(basedir, 'third-party', 'lua*' + os.sep)) + if not ( + False + # LuaJIT 2.0 on macOS requires a CPython linked with "-pagezero_size 10000 -image_base 100000000" + # http://t-p-j.blogspot.com/2010/11/lupa-on-os-x-with-macports-python-26.html + # LuaJIT 2.1-alpha3 fails at runtime. + or (platform == 'darwin' and 'luajit' in os.path.basename(lua_bundle_path.rstrip(os.sep))) + # Let's restrict LuaJIT to x86_64 for now. + or (get_machine() not in ("x86_64", "AMD64") and 'luajit' in os.path.basename(lua_bundle_path.rstrip(os.sep))) + ) + ] if not configs: configs = [ (find_lua_build(no_luajit=option_no_luajit) if not option_use_bundle else {}) @@ -392,6 +635,7 @@ def prepare_extensions(use_cython=True): extra_objects=config.get('extra_objects'), include_dirs=config.get('include_dirs'), define_macros=c_defines, + libraries=['stdc++'] if option_use_luau else [], )) if not use_cython: diff --git a/third-party/luau b/third-party/luau new file mode 160000 index 00000000..0afee000 --- /dev/null +++ b/third-party/luau @@ -0,0 +1 @@ +Subproject commit 0afee00064945abbeb6e6ef2b87633564c54e176