readd default scenarios

Author: chibu

examples/guidellm_example.py

@@ -13,10 +13,10 @@
     target="http://localhost:8000/v1",
     data_type="emulated",
     max_seconds=30,
-    #data="prompt_tokens=512,generated_tokens=256,output_tokens=256",
-    data="prompt_tokens=128,generated_tokens=128,output_tokens=128",
+    #config = "benchmarking_32k",
+    data="prompt_tokens=128,output_tokens=128",
     branch = "update_guidellm",
-    #vllm_kwargs={"enable-chunked-prefill": True}
+    vllm_kwargs={"enable-chunked-prefill": True}
 )
 
 task.execute_remotely("remote-upgrade-default")

setup.py

@@ -7,7 +7,7 @@
     version="0.1.0",
     author="NM MLR",
     description="Automation tools",
-    #url="https://github.com/neuralmagic/research",
+    url="https://github.com/neuralmagic/research",
     package_dir={"": "src"},
     packages=find_packages(
         "src", include=["automation", "automation.*"], exclude=["*.__pycache__.*"]

src/automation/configs.py

@@ -3,3 +3,4 @@
 #DEFAULT_OUTPUT_URI = "gs://neuralmagic-clearml"
 DEFAULT_OUTPUT_URI = "http://10.128.20.60:8081"
 DEFAULT_RESEARCH_BRANCH = "main"
+DEFAULT_GUIDELLM_SCENARIO = "rag"

src/automation/tasks/scripts/guidellm_script.py

@@ -4,6 +4,7 @@
 from automation.utils import resolve_model_id, cast_args, kill_process_tree
 from automation.vllm import start_vllm_server
 from pyhocon import ConfigFactory
+from automation.configs import DEFAULT_GUIDELLM_SCENARIO
 
 def main():
     task = Task.current_task()
@@ -63,8 +64,11 @@ def clean_hocon_value(v):
 
     from pathlib import Path
     from guidellm.benchmark.scenario import GenerativeTextScenario, get_builtin_scenarios
-    filepath = Path(os.path.join(".", "src", "automation", "standards", "benchmarking", "rag.json"))
-    current_scenario = GenerativeTextScenario.from_file(filepath, dict(guidellm_args))
+    if len(get_builtin_scenarios()) > 0:
+        current_scenario = GenerativeTextScenario.from_builtin(DEFAULT_GUIDELLM_SCENARIO, dict(guidellm_args))
+    else:
+        filepath = Path(os.path.join(".", "src", "automation", "standards", "benchmarking", f"{DEFAULT_GUIDELLM_SCENARIO}.json"))
+        current_scenario = GenerativeTextScenario.from_file(filepath, dict(guidellm_args))
     print(current_scenario.model_fields)
     # Start vLLM server
     server_process, server_initialized, server_log = start_vllm_server(

src/automation/vllm/server.py

@@ -55,12 +55,15 @@ def start_vllm_server(
             subprocess_env[k] = str(v)
         else:
             if v == True or v == "True":
-                v = "true"
-            server_command.extend([f"--{k}", str(v)])
+                server_command.append(f"--{k}")
+            else:
+                server_command.extend([f"--{k}", str(v)])
+
 
     server_log_file_name = f"{SERVER_LOG_PREFIX}_{task.id}.txt"
     server_log_file = open(server_log_file_name, "w")
     print("Server command:", " ".join(server_command))
+    print(f"VLLM logs are located at: {server_log_file} in {os.getcwd()}")
     server_process = subprocess.Popen(server_command, stdout=server_log_file, stderr=server_log_file, shell=False, env=subprocess_env)
 
     delay = 5