Add Llama 3 models (#501)

* Add Llama 3 models

* fix

Author: yunfeng-scale

docs/model_zoo.md

@@ -11,6 +11,10 @@ Scale hosts the following models in the LLM Engine Model Zoo:
 | `llama-2-13b-chat`       | ✅                       |                            | text-generation-inference, vllm            | 4096                                           |
 | `llama-2-70b`            | ✅                       | ✅                         | text-generation-inference, vllm            | 4096                                           |
 | `llama-2-70b-chat`       | ✅                       |                            | text-generation-inference, vllm            | 4096                                           |
+| `llama-3-8b`             | ✅                       |                            | vllm                                       | 8192                                           |
+| `llama-3-8b-instruct`    | ✅                       |                            | vllm                                       | 8192                                           |
+| `llama-3-70b`            | ✅                       |                            | vllm                                       | 8192                                           |
+| `llama-3-70b-instruct`   | ✅                       |                            | vllm                                       | 8192                                           |
 | `falcon-7b`              | ✅                       |                            | text-generation-inference, vllm            | 2048                                           |
 | `falcon-7b-instruct`     | ✅                       |                            | text-generation-inference, vllm            | 2048                                           |
 | `falcon-40b`             | ✅                       |                            | text-generation-inference, vllm            | 2048                                           |

model-engine/model_engine_server/domain/use_cases/llm_model_endpoint_use_cases.py

@@ -162,6 +162,10 @@
             "llama-2-13b-chat",
             "llama-2-70b",
             "llama-2-70b-chat",
+            "llama-3-8b",
+            "llama-3-8b-instruct",
+            "llama-3-70b",
+            "llama-3-70b-instruct",
             "falcon-7b",
             "falcon-7b-instruct",
             "falcon-40b",
@@ -231,6 +235,7 @@
     # Can also see 13B, 34B there too
     "gemma": {"max_model_len": 8192, "max_num_batched_tokens": 8192},
     "llama-2": {"max_model_len": None, "max_num_batched_tokens": 4096},
+    "llama-3": {"max_model_len": None, "max_num_batched_tokens": 8192},
     "mistral": {"max_model_len": 8000, "max_num_batched_tokens": 8000},
     "mixtral-8x7b": {"max_model_len": 32768, "max_num_batched_tokens": 32768},
     "mixtral-8x22b": {"max_model_len": 65536, "max_num_batched_tokens": 65536},

model-engine/model_engine_server/infra/repositories/live_tokenizer_repository.py

@@ -40,6 +40,10 @@ def get_default_supported_models_info() -> Dict[str, ModelInfo]:
         "llama-2-13b-chat": ModelInfo("meta-llama/Llama-2-13b-chat-hf", None),
         "llama-2-70b": ModelInfo("meta-llama/Llama-2-70b-hf", None),
         "llama-2-70b-chat": ModelInfo("meta-llama/Llama-2-70b-chat-hf", None),
+        "llama-3-8b": ModelInfo("meta-llama/Meta-Llama-3-8B", None),
+        "llama-3-8b-instruct": ModelInfo("meta-llama/Meta-Llama-3-8B-Instruct", None),
+        "llama-3-70b": ModelInfo("meta-llama/Meta-Llama-3-70B", None),
+        "llama-3-70b-instruct": ModelInfo("meta-llama/Meta-Llama-3-70B-Instruct", None),
         "falcon-7b": ModelInfo("tiiuae/falcon-7b", None),
         "falcon-7b-instruct": ModelInfo("tiiuae/falcon-7b-instruct", None),
         "falcon-40b": ModelInfo("tiiuae/falcon-40b", None),