Improve multiple issues, leaks, improve instrumentation with valgrind, allow python to use debug mode for better debugging as a dependency.

Author: viferga

cmake/CompileOptions.cmake

@@ -68,6 +68,15 @@ set(DEFAULT_INCLUDE_DIRECTORIES)
 # Libraries
 #
 
+# Valgrind
+if(OPTION_TEST_MEMORYCHECK)
+	set(MEMORYCHECK_COMPILE_DEFINITIONS
+		"__MEMORYCHECK__=1"
+	)
+else()
+	set(MEMORYCHECK_COMPILE_DEFINITIONS)
+endif()
+
 # ThreadSanitizer is incompatible with AddressSanitizer and LeakSanitizer
 if(OPTION_BUILD_THREAD_SANITIZER AND (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo"))
 	set(SANITIZER_LIBRARIES -ltsan)
@@ -136,6 +145,7 @@ set(DEFAULT_COMPILE_DEFINITIONS
 	LOG_POLICY_FORMAT_PRETTY=${LOG_POLICY_FORMAT_PRETTY_VALUE}
 	REFLECT_MEMORY_TRACKER=${REFLECT_MEMORY_TRACKER_VALUE}
 	SYSTEM_${SYSTEM_NAME_UPPER}
+	${MEMORYCHECK_COMPILE_DEFINITIONS}
 	${SANITIZER_COMPILE_DEFINITIONS}
 )
 

docker-compose.test.yml

@@ -24,6 +24,7 @@ services:
     image: metacall/core:deps
     build:
       args:
+        METACALL_BUILD_TYPE: ${METACALL_BUILD_TYPE}
         METACALL_INSTALL_OPTIONS: base python ruby netcore7 nodejs typescript file rpc wasm java c cobol go rust rapidjson funchook swig pack backtrace sandbox ${METACALL_BUILD_COVERAGE} # clangformat v8rep51
   dev:
     image: metacall/core:dev

docker-compose.yml

@@ -30,6 +30,7 @@ services:
         METACALL_BASE_IMAGE: $METACALL_BASE_IMAGE
         METACALL_PATH: $METACALL_PATH
         METACALL_TOOLS_PATH: $METACALL_PATH/tools
+        METACALL_BUILD_TYPE: $METACALL_BUILD_TYPE
         METACALL_INSTALL_OPTIONS: base python ruby nodejs typescript file rpc rapidjson funchook swig pack backtrace # clangformat v8rep51 coverage
     environment:
       DEBIAN_FRONTEND: noninteractive

source/cli/metacallcli/source/application.cpp

@@ -337,7 +337,7 @@ bool application::load_path(const char *path, void **handle)
 
 	if (plugin_path == NULL || plugin_extension_handle == NULL)
 	{
-		return NULL;
+		return false;
 	}
 
 	/* Define the cli plugin path as string (core plugin path plus the subpath) */

source/dynlink/source/dynlink_impl_beos.c

@@ -94,7 +94,7 @@ int dynlink_impl_interface_unload_beos(dynlink handle, dynlink_impl impl)
 {
 	(void)handle;
 
-#if defined(__ADDRESS_SANITIZER__) || defined(__THREAD_SANITIZER__) || defined(__MEMORY_SANITIZER__)
+#if defined(__MEMORYCHECK__) || defined(__ADDRESS_SANITIZER__) || defined(__THREAD_SANITIZER__) || defined(__MEMORY_SANITIZER__)
 	/* Disable dlclose when running with address sanitizer in order to maintain stacktraces */
 	(void)impl;
 	return 0;

source/dynlink/source/dynlink_impl_unix.c

@@ -114,8 +114,8 @@ int dynlink_impl_interface_unload_unix(dynlink handle, dynlink_impl impl)
 {
 	(void)handle;
 
-#if defined(__ADDRESS_SANITIZER__) || defined(__THREAD_SANITIZER__) || defined(__MEMORY_SANITIZER__)
-	/* Disable dlclose when running with address sanitizer in order to maintain stacktraces */
+#if defined(__MEMORYCHECK__) || defined(__ADDRESS_SANITIZER__) || defined(__THREAD_SANITIZER__) || defined(__MEMORY_SANITIZER__)
+	/* Disable dlclose when running with valgrind or sanitizers in order to maintain stacktraces */
 	(void)impl;
 	return 0;
 #else

source/loaders/py_loader/source/py_loader_impl.c

@@ -4206,6 +4206,7 @@ void py_loader_impl_sys_path_print(PyObject *sys_path_list)
 
 	log_write("metacall", LOG_LEVEL_DEBUG, sys_path_format_str, sys_path_str);
 
+	Py_XDECREF(sys_path_str_obj);
 	Py_XDECREF(separator);
 }
 #endif

source/ports/js_port/test/main.cpp

@@ -320,7 +320,7 @@ void ModulesClear()
 		 it != modules.end(); ++it)
 	{
 /* Disable dlclose when running with address sanitizer in order to maintain stacktraces */
-#if !defined(__ADDRESS_SANITIZER__) && !defined(__THREAD_SANITIZER__) && !defined(__MEMORY_SANITIZER__)
+#if !defined(__MEMORYCHECK__) && !defined(__ADDRESS_SANITIZER__) && !defined(__THREAD_SANITIZER__) && !defined(__MEMORY_SANITIZER__)
 	#if defined(JS_PORT_TEST_WIN)
 		FreeLibrary(it->second);
 	#elif defined(JS_PORT_TEST_UNIX)

source/serials/rapid_json_serial/source/rapid_json_serial_impl.cpp

@@ -12,6 +12,9 @@
 
 #include <log/log.h>
 
+// TODO: RapidJSON seems to be outdated, but we use it meanwhile there's a better solution.
+// Here's a patch for some of the bugs in the library: https://github.com/Tencent/rapidjson/issues/1928
+
 #include <rapidjson/document.h>
 #include <rapidjson/error/en.h>
 #include <rapidjson/stringbuffer.h>

source/tests/metacall_python_without_env_vars_test/CMakeLists.txt

@@ -115,6 +115,26 @@ target_link_libraries(${target}
 	${DEFAULT_LINKER_OPTIONS}
 )
 
+if(OPTION_BUILD_ADDRESS_SANITIZER OR OPTION_BUILD_THREAD_SANITIZER)
+	# TODO: This test leaks and I am not sure if it is a false positive or not:
+	#
+	# Direct leak of 18682 byte(s) in 12 object(s) allocated from:
+	#     #0 0x7fa978386bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
+	#     #1 0x7fa9734ddbb1 in _PyMem_RawMalloc ../Objects/obmalloc.c:101
+
+	# Indirect leak of 2775 byte(s) in 2 object(s) allocated from:
+	#     #0 0x7fa978386bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
+	#     #1 0x7fa9734ddbb1 in _PyMem_RawMalloc ../Objects/obmalloc.c:101
+
+	# SUMMARY: AddressSanitizer: 21457 byte(s) leaked in 14 allocation(s).
+	#
+	# Valgrind does not show anything:
+	# valgrind --tool=memcheck --leak-check=full --show-leak-kinds=possibly --track-origins=yes --num-callers=500 --suppressions=../source/tests/memcheck/valgrind-python.supp ./metacall-python-without-env-vars-testd
+	# For checking all the leaks (and including false positives), check:
+	# valgrind --tool=memcheck --leak-check=full --show-leak-kinds=all --track-origins=yes --num-callers=500 --suppressions=../source/tests/memcheck/valgrind-python.supp ./metacall-python-without-env-vars-testd
+	return()
+endif()
+
 #
 # Define test
 #