perf: revert preload.

Author: liqiang-fit2cloud

apps/common/management/commands/services/services/gunicorn.py

@@ -19,7 +19,6 @@ def cmd(self):
         cmd = [
             'gunicorn', 'maxkb.wsgi:application',
             '-b', bind,
-            '--preload',
             '-k', 'gthread',
             '--threads', '200',
             '-w', str(self.worker),

apps/common/management/commands/services/services/local_model.py

@@ -29,7 +29,6 @@ def cmd(self):
         cmd = [
             'gunicorn', 'maxkb.wsgi:application',
             '-b', bind,
-            '--preload',
             '-k', 'gthread',
             '--threads', '200',
             '-w', str(worker),

apps/common/management/commands/services/services/scheduler.py

@@ -21,7 +21,6 @@ def cmd(self):
         cmd = [
             'gunicorn', 'maxkb.wsgi:application',
             '-b', bind,
-            '--preload',
             '-k', 'gthread',
             '--threads', '200',
             '-w', str(self.worker),

apps/maxkb/settings/__init__.py

@@ -9,4 +9,5 @@
 from .base import *
 from .logging import *
 from .auth import *
-from .lib import *
+from .lib import *
+from .mem import *

apps/maxkb/settings/mem.py

@@ -0,0 +1,38 @@
+# coding=utf-8
+import os
+import gc
+import threading
+from maxkb.const import CONFIG
+from common.utils.logger import maxkb_logger
+import tracemalloc
+
+CURRENT_PID=os.getpid()
+GC_THRESHOLD = (100, 5, 5)
+# 1 hour
+GC_INTERVAL = 3600
+
+def change_gc_threshold():
+    old_threshold = gc.get_threshold()
+    gc.set_threshold(*GC_THRESHOLD)
+    maxkb_logger.debug(f"(PID: {CURRENT_PID}) GC thresholds changed from {old_threshold} → {GC_THRESHOLD}")
+
+
+def force_gc():
+    snapshot = tracemalloc.take_snapshot()
+    top_stats = snapshot.statistics('lineno')
+    maxkb_logger.debug("[ Top 10 memory-consuming lines ]")
+    for stat in top_stats[:10]:
+        maxkb_logger.debug(stat)
+    collected = gc.collect()
+    maxkb_logger.debug(f"(PID: {CURRENT_PID}) Forced GC ({collected} objects collected)")
+    threading.Timer(GC_INTERVAL, force_gc).start()
+
+
+def init_memory_optimization():
+    tracemalloc.start()
+    change_gc_threshold()
+    force_gc()
+    maxkb_logger.debug("(PID: {CURRENT_PID}) Memory optimization (GC tuning) started.")
+
+if CONFIG.get("ENABLE_MEMORY_OPTIMIZATION", '1') == "1":
+    init_memory_optimization()

installer/Dockerfile-base

@@ -31,7 +31,6 @@ RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
     chmod g+xr /usr/bin/ld.so && \
     chmod g+x /usr/local/bin/python* && \
     apt-get clean all && \
-    echo "/usr/lib/$(uname -m)-linux-gnu/libjemalloc.so.2" > /etc/ld.so.preload && \
     rm -rf /var/lib/apt/lists/* /usr/share/doc/* /usr/share/man/* /usr/share/info/* /usr/share/locale/* /usr/share/lintian/* /usr/share/linda/* /var/cache/* /var/log/* /var/tmp/* /tmp/*
 COPY --from=vector-model --chmod=700 /opt/maxkb-app/model /opt/maxkb-app/model