profiler & input: Add backpressure by lowering maxsize of profiler queue to avoid OOM kills

Author: AlyaGomaa

managers/process_manager.py

@@ -53,7 +53,7 @@ def __init__(self, main):
         # to pass flows to the profiler
         # this max size is decided based on the avg size of each flow and
         # tha max memory (4g) that this queue is allowed to use
-        self.profiler_queue = Queue(maxsize=5162220)
+        self.profiler_queue = Queue(maxsize=50000)
         self.termination_event: Event = Event()
         # to make sure we only warn the user once about
         # the pending modules

slips_files/core/profiler.py

@@ -101,11 +101,12 @@ def init(
         self.stop_profiler_workers_event = multiprocessing.Event()
         # each msg received from inputprocess will be put here, and each one
         # profiler worker will retrieve msgs from this queue.
-        # the goal of this is to have main() handle the stop msg.
+        # the goal of this q is to have main() handle the stop msg.
         # so without this, only 1 of the 3 workers receives the stop msg
         # and exits, and the rest of the 2 workers AND the main() keep
         # waiting for new msgs
-        self.flows_to_process_q = multiprocessing.Queue(maxsize=5162220)
+
+        self.flows_to_process_q = multiprocessing.Queue(maxsize=50000)
         self.handle_setting_local_net_lock = multiprocessing.Lock()
         self.is_first_msg = True
         # runs a separate server process behind the scenes.