feat: use "MODEL_NAME" instead of "MODEL_PATH"

Author: TimPietruskyRunPod

.runpod/hub.json

@@ -12,7 +12,7 @@
     "allowedCudaVersions": ["12.7", "12.6", "12.5", "12.4"],
     "env": [
       {
-        "key": "MODEL_PATH",
+        "key": "MODEL_NAME",
         "input": {
           "name": "Model",
           "type": "huggingface",

Dockerfile

@@ -18,14 +18,14 @@ COPY handler.py engine.py utils.py download_model.py test_input.json ./
 COPY public/ ./public/
 
 # Setup for Option 2: Building the Image with the Model included
-ARG MODEL_PATH=""
+ARG MODEL_NAME=""
 ARG TOKENIZER_NAME=""
 ARG BASE_PATH="/runpod-volume"
 ARG QUANTIZATION=""
 ARG MODEL_REVISION=""
 ARG TOKENIZER_REVISION=""
 
-ENV MODEL_PATH=$MODEL_PATH \
+ENV MODEL_NAME=$MODEL_NAME \
     MODEL_REVISION=$MODEL_REVISION \
     TOKENIZER_NAME=$TOKENIZER_NAME \
     TOKENIZER_REVISION=$TOKENIZER_REVISION \
@@ -42,7 +42,7 @@ RUN --mount=type=secret,id=HF_TOKEN,required=false \
     if [ -f /run/secrets/HF_TOKEN ]; then \
         export HF_TOKEN=$(cat /run/secrets/HF_TOKEN); \
     fi && \
-    if [ -n "$MODEL_PATH" ]; then \
+    if [ -n "$MODEL_NAME" ]; then \
         python3 download_model.py; \
     fi
 

README.md

@@ -14,7 +14,7 @@ All behaviour is controlled through environment variables:
 
 | Environment Variable              | Description                                       | Default                               | Options                                                                                   |
 | --------------------------------- | ------------------------------------------------- | ------------------------------------- | ----------------------------------------------------------------------------------------- |
-| `MODEL_PATH`                      | Path of the model weights                         | "meta-llama/Meta-Llama-3-8B-Instruct" | Local folder or Hugging Face repo ID                                                      |
+| `MODEL_NAME`                      | Hugging Face model name or local path             | "meta-llama/Meta-Llama-3-8B-Instruct" | Hugging Face repo ID or local folder path                                                 |
 | `HF_TOKEN`                        | HuggingFace access token for gated/private models |                                       | Your HuggingFace access token                                                             |
 | `TOKENIZER_PATH`                  | Path of the tokenizer                             |                                       |                                                                                           |
 | `TOKENIZER_MODE`                  | Tokenizer mode                                    | "auto"                                | "auto", "slow"                                                                            |

docker-compose.yml

@@ -14,7 +14,7 @@ services:
     environment:
       - HOST=0.0.0.0
       - PORT=30000
-      - MODEL_PATH=meta-llama/Llama-3.2-1B-Instruct
+      - MODEL_NAME=meta-llama/Llama-3.2-1B-Instruct
       - TRUST_REMOTE_CODE=true
       - ATTENTION_BACKEND=flashinfer
       - SAMPLING_BACKEND=flashinfer

docs/conventions.md

@@ -45,7 +45,7 @@
 feat(docker): add github workflow for automated builds
 fix(handler): resolve openai compatibility issue
 docs(readme): update installation instructions
-refactor(engine): migrate from MODEL_NAME to MODEL_PATH
+refactor(engine): migrate from MODEL_PATH to MODEL_NAME
 chore(deps): update requirements.txt
 ```
 

download_model.py

@@ -17,7 +17,7 @@ def setup_env():
         os.environ.update(
             {
                 "HF_HOME": f"{BASE_DIR}/hf_cache",
-                "MODEL_PATH": "openchat/openchat-3.5-0106",
+                "MODEL_NAME": "openchat/openchat-3.5-0106",
                 "HF_HUB_ENABLE_HF_TRANSFER": "1",
                 "TENSORIZE": "1",
                 "TENSORIZER_NUM_GPUS": "1",
@@ -52,7 +52,7 @@ def download(name, revision, type, cache_dir):
     setup_env()
     cache_dir = os.getenv("HF_HOME")
     model_name, model_revision = (
-        os.getenv("MODEL_PATH"),
+        os.getenv("MODEL_NAME"),
         os.getenv("MODEL_REVISION") or None,
     )
     tokenizer_name, tokenizer_revision = (
@@ -63,7 +63,7 @@ def download(name, revision, type, cache_dir):
     model_path_downloaded = download(model_name, model_revision, "model", cache_dir)
 
     metadata = {
-        "MODEL_PATH": model_path_downloaded,
+        "MODEL_NAME": model_path_downloaded,
         "MODEL_REVISION": os.getenv("MODEL_REVISION"),
         "QUANTIZATION": os.getenv("QUANTIZATION"),
     }

engine.py

@@ -10,7 +10,7 @@
 class SGlangEngine:
     def __init__(
         self,
-        model=os.getenv("MODEL_PATH"),
+        model=os.getenv("MODEL_NAME"),
         host=os.getenv("HOST", "0.0.0.0"),
         port=int(os.getenv("PORT", 30000)),
     ):
@@ -33,7 +33,7 @@ def start_server(self):
 
         # Dictionary of all possible options and their corresponding env var names
         options = {
-            "MODEL_PATH": "--model-path",
+            "MODEL_NAME": "--model-path",
             "TOKENIZER_PATH": "--tokenizer-path",
             "TOKENIZER_MODE": "--tokenizer-mode",
             "LOAD_FORMAT": "--load-format",