update max processes limit

Author: LuckyYC

docker-compose.dev.yml

@@ -59,19 +59,16 @@ services:
       mysql:
         condition: service_healthy
     environment:
-      - DB_HOST=mysql
-      - DB_PORT=3306
-      - DB_USER=root
-      - DB_PASSWORD=lmeterx123
-      - DB_NAME=lmeterx
-      - DOCKER_CONTAINER=true
-      - TERM=dumb
-      - PYTHONUNBUFFERED=1
+      DB_HOST: mysql
+      DB_PORT: 3306
+      DB_USER: root
+      DB_PASSWORD: lmeterx123
+      DB_NAME: lmeterx
       # Multi-process configuration
-      - ENABLE_MULTIPROCESS=auto
-      - LOCUST_CPU_CORES=2.0                    # Unified CPU core specification (matches deploy.resources.limits.cpus)
-      - MULTIPROCESS_THRESHOLD=1000            # Enable multiprocess when users >= 1000
-      - MIN_USERS_PER_PROCESS=100               # Each process handles at least 100 users
+      ENABLE_MULTIPROCESS: auto
+      LOCUST_CPU_CORES: 2.0
+      MULTIPROCESS_THRESHOLD: 1000
+      MIN_USERS_PER_PROCESS: 100             # Each process handles at least 100 users
     volumes:
       - ./logs:/logs
       - ./upload_files:/app/upload_files

docker-compose.yml

@@ -53,19 +53,16 @@ services:
       mysql:
         condition: service_healthy
     environment:
-      - DB_HOST=mysql
-      - DB_PORT=3306
-      - DB_USER=root
-      - DB_PASSWORD=lmeterx123
-      - DB_NAME=lmeterx
-      - DOCKER_CONTAINER=true
-      - TERM=dumb
-      - PYTHONUNBUFFERED=1
+      DB_HOST: mysql
+      DB_PORT: 3306
+      DB_USER: root
+      DB_PASSWORD: lmeterx123
+      DB_NAME: lmeterx
       # Multi-process configuration
-      - ENABLE_MULTIPROCESS=auto
-      - LOCUST_CPU_CORES=2.0                    # Unified CPU core specification (matches deploy.resources.limits.cpus)
-      - MULTIPROCESS_THRESHOLD=1000             # Enable multiprocess when users >= 1000
-      - MIN_USERS_PER_PROCESS=100               # Each process handles at least 100 users
+      ENABLE_MULTIPROCESS: auto
+      LOCUST_CPU_CORES: 2.0
+      MULTIPROCESS_THRESHOLD: 1000
+      MIN_USERS_PER_PROCESS: 100
     volumes:
       - ./logs:/logs
       - ./upload_files:/app/upload_files
@@ -75,7 +72,7 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '2.0'                           # Must match LOCUST_CPU_CORES
+          cpus: '2.0'
           memory: 2G
         reservations:
           cpus: '1.0'

st_engine/config/multiprocess.py

@@ -27,10 +27,10 @@ def config(self) -> Dict[str, Any]:
                 "cpu_cores": os.environ.get("LOCUST_CPU_CORES", ""),
                 "processes": os.environ.get("LOCUST_PROCESSES", ""),
                 "multiprocess_threshold": int(
-                    os.environ.get("MULTIPROCESS_THRESHOLD", "5")
+                    os.environ.get("MULTIPROCESS_THRESHOLD", "1000")
                 ),
                 "min_users_per_process": int(
-                    os.environ.get("MIN_USERS_PER_PROCESS", "5")
+                    os.environ.get("MIN_USERS_PER_PROCESS", "100")
                 ),
                 "force_single_process": os.environ.get(
                     "FORCE_SINGLE_PROCESS", "false"
@@ -80,7 +80,7 @@ def _detect_cpu_count(self) -> int:
 
             # Respect container limits in Docker environment
             if self.is_docker:
-                return min(system_cpu_count, 8)  # Limit to 8 processes for safety
+                return min(system_cpu_count, 4)  # Limit to 4 processes for safety
 
             return system_cpu_count
         except (ImportError, NotImplementedError):
@@ -238,12 +238,12 @@ def get_fallback_queue_type() -> str:
 
 # Performance tuning parameters, clearer names
 MULTIPROCESS_THRESHOLD = int(
-    os.environ.get("MULTIPROCESS_THRESHOLD", "5")
+    os.environ.get("MULTIPROCESS_THRESHOLD", "1000")
 )  # Min users to enable multiprocess (must be > 1000)
 
 MIN_USERS_PER_PROCESS = int(
-    os.environ.get("MIN_USERS_PER_PROCESS", "5")
-)  # Min users each process should handle (updated from 500)
+    os.environ.get("MIN_USERS_PER_PROCESS", "100")
+)  # Min users each process should handle
 
 
 __all__ = [

st_engine/engine/core.py

@@ -66,6 +66,7 @@ class GlobalConfig:
     cert_config: Optional[Union[str, Tuple[str, str]]] = None
     field_mapping: Optional[str] = None
     test_data: Optional[str] = None
+    duration: int = 60
 
 
 @dataclass

st_engine/engine/locustfile.py

@@ -12,7 +12,8 @@
 import sys
 import tempfile
 import time
-from typing import Any, Dict, Optional, Tuple
+from dataclasses import field
+from typing import Any, Dict, Optional, Tuple, Union
 
 import urllib3
 from gevent import queue
@@ -24,6 +25,7 @@
 
 from config.base import (
     DEFAULT_API_PATH,
+    DEFAULT_CONTENT_TYPE,
     DEFAULT_PROMPT,
     DEFAULT_TIMEOUT,
     DEFAULT_WAIT_TIME_MAX,
@@ -218,6 +220,12 @@ def init_parser(parser):
         default="",
         help="Custom test data in JSONL format or file path",
     )
+    parser.add_argument(
+        "--duration",
+        type=int,
+        default=60,
+        help="Test duration in seconds (used as fallback when start_time is unavailable)",
+    )
 
 
 @events.init.add_listener
@@ -256,6 +264,7 @@ def on_locust_init(environment, **kwargs):
         global_config.key_file = options.key_file
         global_config.field_mapping = options.field_mapping
         global_config.test_data = options.test_data
+        global_config.duration = int(options.duration)
 
         # Parse and validate configuration
         global_config.headers = ConfigManager.parse_headers(
@@ -420,9 +429,35 @@ def on_test_stop(environment, **kwargs):
     global_config = get_global_config()
     task_id = global_config.task_id
     task_logger = GlobalStateManager.get_task_logger(task_id)
+
+    execution_time = None
     start_time = GlobalStateManager.get_start_time()
     end_time = time.time()
-    execution_time = max(end_time - start_time, 0.001)  # avoid division by zero
+
+    try:
+        if start_time is not None and end_time is not None:
+            execution_time = max(end_time - start_time, 0.001)
+        else:
+            duration = getattr(global_config, "duration", None)
+            if duration is not None and duration > 0:
+                execution_time = float(duration)
+            else:
+                execution_time = 60.0
+                task_logger.error(
+                    f"Failed to get effective execution time: start_time={start_time}, duration={duration}"
+                )
+
+                raise ValueError(
+                    f"Failed to get effective execution time: start_time={start_time}, duration={duration}"
+                )
+
+    except Exception as e:
+        task_logger.error(
+            "Failed to calculate execution time: %s" % str(e), exc_info=True
+        )
+        execution_time = 60.0
+        raise RuntimeError("Failed to calculate execution time: %s" % str(e))
+
     from utils.common import calculate_custom_metrics, get_locust_stats
 
     # Check if this is a worker process
@@ -461,7 +496,8 @@ def on_test_stop(environment, **kwargs):
 
     task_logger.debug(
         f"Process type: {'Worker' if is_worker else 'Master'}, "
-        f"Multi-process: {is_multiprocess}, Worker count: {worker_count}"
+        f"Multi-process: {is_multiprocess}, Worker count: {worker_count}, "
+        f"Execution time: {execution_time}"
     )
 
     # === WORKER PROCESS LOGIC ===

st_engine/engine/runner.py

@@ -296,9 +296,6 @@ def read_output(pipe, lines, stream_name):
         # This includes the task duration, Locust's own stop timeout, and an extra buffer.
         locust_stop_timeout_config = 99
         wait_timeout_total = task_duration_seconds + locust_stop_timeout_config + 30
-        task_logger.info(
-            f"Waiting for Locust process {process.pid} to complete with a timeout of {wait_timeout_total} seconds."
-        )
 
         try:
             process.wait(timeout=wait_timeout_total)