Merge branch 'feature/memory_leak_testing' into firmware_version_upgrade

Author: aljazdu

bindings/python/src/DeviceBindings.cpp

@@ -607,6 +607,13 @@ void DeviceBindings::bind(pybind11::module& m, void* pCallstack) {
                 return d.getLeonMssCpuUsage();
             },
             DOC(dai, DeviceBase, getLeonMssCpuUsage))
+        .def(
+            "getProcessMemoryUsage",
+            [](DeviceBase& d) {
+                py::gil_scoped_release release;
+                return d.getProcessMemoryUsage();
+            },
+            DOC(dai, DeviceBase, getProcessMemoryUsage))
         .def(
             "addLogCallback",
             [](DeviceBase& d, std::function<void(LogMessage)> callback) {

include/depthai/device/DeviceBase.hpp

@@ -592,6 +592,13 @@ class DeviceBase {
      */
     CpuUsage getLeonMssCpuUsage();
 
+    /**
+     * Retrieves current Rss memory usage of the device process
+     *
+     * @returns Current Rss memory used
+     */
+    int64_t getProcessMemoryUsage();
+
     /**
      * Check if EEPROM is available
      * @returns True if EEPROM is present on board, false otherwise

src/device/DeviceBase.cpp

@@ -1241,6 +1241,10 @@ CpuUsage DeviceBase::getLeonMssCpuUsage() {
     return pimpl->rpcClient->call("getLeonMssCpuUsage").as<CpuUsage>();
 }
 
+int64_t DeviceBase::getProcessMemoryUsage() {
+    return pimpl->rpcClient->call("getProcessMemoryUsage").as<int64_t>();
+}
+
 UsbSpeed DeviceBase::getUsbSpeed() {
     return pimpl->rpcClient->call("getUsbSpeed").as<UsbSpeed>();
 }

tests/stability/stability_test_depthai.py

@@ -3,6 +3,9 @@
 import time
 from typing import List, Dict
 import datetime
+import sys
+
+MEMORY_LEAK_DETECTION_THRESHOLD = 1.05
 
 def stability_test(fps):
     # Creates the pipeline and a default device implicitly
@@ -66,6 +69,11 @@ def stability_test(fps):
         print("Starting the stability test...")
         tStart = time.time()
         p.start()
+
+        # Delay so the device process finishes starting up 
+        time.sleep(10)
+        initialProcessMemoryUsage = p.getDefaultDevice().getProcessMemoryUsage()
+        print(f"Initial depthai-device process memory usage is: {initialProcessMemoryUsage} kB")
         while True:
             for name, queue in benchmarkReportQueues.items():
                 report = queue.get(timeout=datetime.timedelta(minutes=1)) # 1 minute timeout
@@ -77,6 +85,12 @@ def stability_test(fps):
                 else:
                     raise RuntimeError(f"Timeout reached for {name} benchmark report")
                 queue.tryGetAll() # Clear the queue
+
+            # Detect memory leaks
+            processMemoryUsage = p.getDefaultDevice().getProcessMemoryUsage()
+            if processMemoryUsage > MEMORY_LEAK_DETECTION_THRESHOLD * initialProcessMemoryUsage:
+                raise RuntimeError("Memory used by depthai-device process increased above the given threshold - potential memory leak detected")
+            print(f"Memory used by depthai-device process: {processMemoryUsage} kB. Current time: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())}")
             print(f"Running for {datetime.timedelta(seconds=time.time() - tStart)}", flush=True)
 
 if __name__ == "__main__":