Skip to content

Commit d79e9a8

Browse files
authored
Cache size to 0 to improve performance on LLMs (#490)
1 parent 6f9bbfa commit d79e9a8

File tree

2 files changed

+1
-2
lines changed

2 files changed

+1
-2
lines changed

ai_ref_kits/agentic_multimodal_travel_planer/download_and_run_models_Windows.bat

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ REM Start LLM service
9393
REM --port = gRPC, --rest_port = HTTP REST (chat/completions). Agents use HTTP, so REST must be on LLM_PORT.
9494
echo Starting LLM service (REST on %LLM_PORT%, gRPC on 8011)...
9595
set LLM_GRPC_PORT=8011
96-
set LLM_ARGS=--port %LLM_GRPC_PORT% --rest_port %LLM_PORT% --model_repository_path "%MODELS_DIR%" --source_model "%LLM_MODEL%" --tool_parser hermes3 --cache_size 2 --task text_generation --enable_prefix_caching true
96+
set LLM_ARGS=--port %LLM_GRPC_PORT% --rest_port %LLM_PORT% --model_repository_path "%MODELS_DIR%" --source_model "%LLM_MODEL%" --tool_parser hermes3 --cache_size 0 --task text_generation
9797
if not "%TARGET_DEVICE%"=="" set LLM_ARGS=%LLM_ARGS% --target_device %TARGET_DEVICE%
9898
REM Use PowerShell Start-Process to launch detached
9999
powershell -Command "Start-Process -FilePath '%OVMS_PATH%' -ArgumentList '%LLM_ARGS%' -RedirectStandardOutput '%LOGS_DIR%\ovms_llm.log' -RedirectStandardError '%LOGS_DIR%\ovms_llm.err' -WindowStyle Hidden" || (echo Failed to start LLM service && exit /b 1)

ai_ref_kits/agentic_multimodal_travel_planer/download_and_run_models_linux.sh

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,6 @@ if ! docker run -d \
427427
--source_model "${LLM_MODEL}" \
428428
--task text_generation \
429429
--tool_parser ${LLM_TOOL_PARSER} \
430-
${LLM_REASONING_PARSER:+--reasoning_parser ${LLM_REASONING_PARSER}} \
431430
--log_level DEBUG \
432431
${LLM_TARGET_DEVICE_ARG} \
433432
>/dev/null; then

0 commit comments

Comments
 (0)