luzer: fix memory leaks

The patch fixes memory leaks in luzer that manifest themselves in
different ways:

1. `LLVMFuzzerRunDriver()` returns due to error with dictionary.
2. Lua objects accumulate memory between `TestOneInput()` runs,

but the cause is the same for everyone: Lua is a language with
GC-based memory management, and after finishing the work, we
did not free the memory occupied by Lua objects, so libFuzzer
thinks there was a memory leak.

Fixes #52
Fixes #65

Author: ligurio

CHANGELOG.md

@@ -57,3 +57,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - A memory leak in a Lua-based implementation of `TestOneInput()`.
 - An initial buffer size in FuzzedDataProvider.
 - Search of the archiver tool (`ar`) in the OSS Fuzz environment.
+- Memory leak in FuzzedDataProvider (#52).
+- Segfault on parsing a broken dictionary (#65).

docs/usage.md

@@ -228,9 +228,33 @@ and can be disabled by setting the enviroment variable
 `DISABLE_LUAJIT_METRICS`. Learn more about the enviroment variable
 in the section [LuaJIT Metrics](#luajit-metrics).
 
+### Memory leaks
+
+When a target application (or a fuzzer) consumes increasing
+amounts of RAM over time without releasing it, it can be a normal
+memory consumption or memory leak is occurring. The common causes
+of memory leak are: unreleased references, improper handling of
+native resources in Lua. If you are encountering a memory leak in
+a target application while using luzer, you may need to use memory
+debugging tools to identify the specific code segment that isn't
+freeing memory.
+
+luzer can encounter false positive memory leaks during testing.
+When fuzzing native extensions, using FDP (FuzzingDataProvider), or
+using FFI memory leak detection (ASan) should be disabled to
+prevent false reports. Set flag `-detect_leaks=0` using enviroment
+variable [`ASAN_OPTIONS`][asan-flags] as it is recommended by
+AddressSanitizer:
+
+```
+SUMMARY: AddressSanitizer: 96 byte(s) leaked in 6 allocation(s).
+INFO: to ignore leaks on libFuzzer side use -detect_leaks=0.
+```
+
 [ffi-library-url]: https://luajit.org/ext_ffi.html
 [programming-in-lua-8]: https://www.lua.org/pil/8.html
 [programming-in-lua-24]: https://www.lua.org/pil/24.html
 [atheris-native-extensions]: https://github.com/google/atheris/blob/master/native_extension_fuzzing.md
 [atheris-native-extensions-video]: https://www.youtube.com/watch?v=oM-7lt43-GA
 [luacov-website]: https://lunarmodules.github.io/luacov/
+[asan-flags]: https://github.com/google/sanitizers/wiki/addresssanitizerflags

luzer/luzer.c

@@ -437,6 +437,33 @@ free_argv(int argc, char **argv)
 	free(argv);
 }
 
+NO_SANITIZE static void
+shutdown_lua(void)
+{
+	lua_State *L = get_global_lua_state();
+	luaL_cleanup(L);
+	lua_close(L);
+	set_global_lua_state(NULL);
+}
+
+int atexit_retcode;
+
+NO_SANITIZE void
+atexit_handler(void) {
+	_exit(atexit_retcode);
+}
+
+NO_SANITIZE static void
+graceful_exit(int retcode, bool prevent_crash_report) {
+	if (prevent_crash_report) {
+		/* Disable libFuzzer's atexit(). */
+		atexit_retcode = retcode;
+		atexit(&atexit_handler);
+	}
+	shutdown_lua();
+	exit(retcode);
+}
+
 NO_SANITIZE static int
 luaL_fuzz(lua_State *L)
 {
@@ -536,14 +563,8 @@ luaL_fuzz(lua_State *L)
 	jit_status = luajit_has_enabled_jit(L);
 #endif
 	set_global_lua_state(L);
-	int rc = LLVMFuzzerRunDriver(&argc, &argv, &TestOneInput);
-
-	free_argv(argc, argv);
-	luaL_cleanup(L);
-
-	lua_pushnumber(L, rc);
-
-	return 1;
+	graceful_exit(LLVMFuzzerRunDriver(&argc, &argv, &TestOneInput), true);
+	return 0;
 }
 
 static const struct luaL_Reg Module[] = {

luzer/tests/CMakeLists.txt

@@ -371,18 +371,6 @@ if (LUA_HAS_JIT)
     "${TEST_ENV};FFI_LIB_NAME=testlib${CMAKE_SHARED_LIBRARY_SUFFIX}"
     "Done 10 runs in 0 second"
   )
-  string(JOIN ";" TEST_ENVIRONMENT
-    "${TEST_ENV}"
-    LD_PRELOAD=${ASAN_DSO_PATH}
-    FFI_LIB_NAME=testlib_asan${CMAKE_SHARED_LIBRARY_SUFFIX}
-  )
-  # XXX: Memory leak in FDP is expected on Linux, should be fixed
-  # in [1].
-  # 1. https://github.com/ligurio/luzer/issues/52
-  set(PASS_PATTERN "LeakSanitizer: detected memory leaks")
-  if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
-    set(PASS_PATTERN "Done 10 runs in 0 second")
-  endif()
   generate_ffi_test(luzer_ffi_asan
     "${TEST_ENVIRONMENT}"
     ${PASS_PATTERN}
@@ -391,6 +379,8 @@ if (LUA_HAS_JIT)
     "${TEST_ENV}"
     LD_PRELOAD=${UBSAN_DSO_PATH}
     FFI_LIB_NAME=testlib_ubsan${CMAKE_SHARED_LIBRARY_SUFFIX}
+    "${TEST_ENV};LD_PRELOAD=${ASAN_DSO_PATH};FFI_LIB_NAME=testlib_asan.so"
+    "Done 10 runs in [0-9] second"
   )
   generate_ffi_test(luzer_ffi_ubsan
     "${TEST_ENVIRONMENT}"
@@ -419,3 +409,37 @@ if (LUA_HAS_JIT)
     "runtime error: load of null pointer of type"
   )
 endif()
+
+string(JOIN ";" TEST_ENV
+  "LUA_CPATH=${LUA_CPATH}"
+  "LUA_PATH=${LUA_PATH}"
+  "LD_PRELOAD=${ASAN_DSO_PATH}"
+)
+add_test(
+  NAME luzer_missed_dict_test
+  COMMAND ${LUA_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_options_3.lua
+          -dict=${CMAKE_CURRENT_SOURCE_DIR}/nonexistent.dict
+)
+list(APPEND FAIL_REGULAR_EXPRESSION
+  "LeakSanitizer: detected memory leaks"
+  "[0-9]+ byte(s) leaked in [0-9]+ allocation"
+)
+set_tests_properties(luzer_missed_dict_test PROPERTIES
+  ENVIRONMENT "${TEST_ENV}"
+  PASS_REGULAR_EXPRESSION "ParseDictionaryFile: file does not exist or is empty"
+  FAIL_REGULAR_EXPRESSION "${FAIL_REGULAR_EXPRESSION}"
+)
+
+add_test(
+  NAME luzer_leak_memory_test
+  COMMAND ${LUA_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/leak_memory.lua
+          -rss_limit_mb=500
+)
+list(APPEND PASS_REGULAR_EXPRESSION
+  "libFuzzer disabled leak detection after every mutation"
+  "ERROR: libFuzzer: out-of-memory"
+)
+set_tests_properties(luzer_leak_memory_test PROPERTIES
+  ENVIRONMENT "${TEST_ENV}"
+  PASS_REGULAR_EXPRESSION "${PASS_REGULAR_EXPRESSION}"
+)

luzer/tests/leak_memory.lua

@@ -0,0 +1,21 @@
+local luzer = require("luzer")
+
+local leaky_cache = {} -- luacheck: no unused
+
+-- Function to create a large, nested table.
+local function make_big_object(size)
+    if size <= 0 then
+        return {}
+    end
+    -- Create nested tables recursively.
+    return {
+        make_big_object(size - 1)
+    }
+end
+
+local function TestOneInput(_buf)
+    local new_object = make_big_object(2)
+    table.insert(leaky_cache, new_object)
+end
+
+luzer.Fuzz(TestOneInput)

luzer/tests/test_options_3.lua

@@ -0,0 +1,10 @@
+local luzer = require("luzer")
+
+local function TestOneInput(buf)
+    local fdp = luzer.FuzzedDataProvider(buf)
+    local str = fdp:consume_string(100)
+    local str_chars = {}
+    str:gsub(".", function(c) table.insert(str_chars, c) end)
+end
+
+luzer.Fuzz(TestOneInput)