Add shm size check

lightllm/server/api_cli.py

@@ -187,6 +187,9 @@ def make_argument_parser() -> argparse.ArgumentParser:
     )
     parser.add_argument("--disable_log_stats", action="store_true", help="disable logging throughput stats.")
     parser.add_argument("--log_stats_interval", type=int, default=10, help="log stats interval in second.")
+    parser.add_argument(
+        "--disable-shm-warning", action="store_true", default=False, help="Disable periodic shm warning logs"
+    )
 
     parser.add_argument("--router_token_ratio", type=float, default=0.0, help="token ratio to control router dispatch")
     parser.add_argument(

lightllm/server/api_start.py

@@ -4,6 +4,7 @@
 import uuid
 import subprocess
 import signal
+import shutil
 from lightllm.utils.net_utils import alloc_can_use_network_port, PortLocker
 from lightllm.utils.start_utils import process_manager, kill_recursive
 from .metrics.manager import start_metric_manager
@@ -19,6 +20,37 @@
 logger = init_logger(__name__)
 
 
+def get_shm_size_gb():
+    """
+    获取 /dev/shm 的总大小（以GB为单位）。
+    """
+    try:
+        shm_path = "/dev/shm"
+        if not os.path.exists(shm_path):
+            logger.error(f"{shm_path} not exist, this may indicate a system or Docker configuration anomaly.")
+            return 0
+
+        # shutil.disk_usage 返回 (total, used, free)
+        total_bytes = shutil.disk_usage(shm_path).total
+        total_gb = total_bytes / (1024 ** 3)
+        return total_gb
+    except Exception as e:
+        logger.error(f"Error getting /dev/shm size: {e}")
+        return 0
+
+
+def check_shm_size():
+    RED = "\033[91m"
+    GREEN = "\033[92m"
+    ENDC = "\033[0m"
+    shm_size = get_shm_size_gb()
+    required_size = 128  # 128G
+    if shm_size < required_size:
+        logger.warning(f"{RED}Available shm size is less than 128G: {shm_size:.2f}G{ENDC}")
+    else:
+        logger.info(f"{GREEN}/dev/shm available space is sufficient ({shm_size:.2f} GB >= {required_size} GB).{ENDC}")
+
+
 def setup_signal_handlers(http_server_process, process_manager):
     def signal_handler(sig, frame):
         if sig == signal.SIGINT:
@@ -62,6 +94,19 @@ def signal_handler(sig, frame):
 def normal_or_p_d_start(args):
     set_unique_server_name(args)
 
+    check_shm_size()
+
+    if not args.disable_shm_warning:
+        import threading
+
+        def periodic_shm_warning():
+            while True:
+                check_shm_size()
+                time.sleep(120)  # 每 120 秒打印一次警告日志
+
+        shm_warning_thread = threading.Thread(target=periodic_shm_warning, daemon=True)
+        shm_warning_thread.start()
+
     if args.enable_mps:
         from lightllm.utils.device_utils import enable_mps