mgr/rest: Trim request array and limit size

Presently, the requests array in the REST module has the potential to grow
indefinitely, leading to excessive memory consumption, particularly when
dealing with lengthy and intricate request results.

To address this issue, a limit will be imposed on the requests array within
the REST module.
This limitation will be governed by the `mgr/restful/x/max_requests` configuration
parameter specific to the REST module.
when submit_request called we will check request array if exceed max_request option
if it does we will check if the future trimmed request finished and log error
message in case we are trimming un-finished requests.

Fixes: https://tracker.ceph.com/issues/59580
Signed-off-by: Nitzan Mordechai <[email protected]>

Author: NitzanMordhai

src/pybind/mgr/restful/module.py

@@ -12,6 +12,7 @@
 import traceback
 import socket
 import fcntl
+from typing import cast
 
 from . import common
 from . import context
@@ -197,6 +198,7 @@ class Module(MgrModule):
         {'name': 'server_port'},
         {'name': 'key_file'},
         {'name': 'enable_auth', 'type': 'bool', 'default': True},
+        {'name': 'max_requests', 'type': 'int', 'default': 500},
     ]
 
     COMMANDS = [
@@ -243,6 +245,7 @@ def __init__(self, *args, **kwargs):
 
         self.stop_server = False
         self.serve_event = threading.Event()
+        self.max_requests = cast(int, self.get_localized_module_option('max_requests', 500))
 
 
     def serve(self):
@@ -599,6 +602,16 @@ def submit_request(self, _request, **kwargs):
         with self.requests_lock:
             request = CommandsRequest(_request)
             self.requests.append(request)
+            if len(self.requests) > self.max_requests:
+                req_to_trim = 0
+                for i, req in enumerate(self.requests):
+                    if req.is_finished():
+                        self.log.error("Trimmed one finished request due to exceeded maximum requests limit")
+                        req_to_trim = i
+                        break
+                    else:
+                        self.log.error("Trimmed the oldest unfinished request due to exceeded maximum requests limit")
+                self.requests.pop(req_to_trim)
         if kwargs.get('wait', 0):
             while not request.is_finished():
                 time.sleep(0.001)