🚧 WIP

Author: Dmytro Parfeniuk

src/guidellm/backend/base.py

@@ -309,6 +309,7 @@ def _cachable_default_model(backend: Backend) -> str:
     :rtype: str
     :raises ValueError: If no models are available.
     """
+
     logger.debug("Getting default model for backend: {}", backend)
     models = backend.available_models()
     if models:

src/guidellm/backend/openai.py

@@ -103,6 +103,7 @@ async def make_request(
 
         request_args.update(self._request_args)
 
+        print(">>> Creating stream object for OpenAI server ")
         stream = await self._async_client.chat.completions.create(
             model=self.model,
             messages=[
@@ -111,8 +112,10 @@ async def make_request(
             stream=True,
             **request_args,
         )
+
         token_count = 0
         async for chunk in stream:
+            print(f"Getting chunk: {chunk}")
             choice = chunk.choices[0]
             token = choice.delta.content or ""
 
@@ -145,6 +148,9 @@ def available_models(self) -> List[str]:
         :raises openai.OpenAIError: If an error occurs while retrieving models.
         """
 
+        # TODO: Remove this line
+        return ["Meta-Llama-3-8B.Q4_K_M.gguf"]
+
         try:
             return [model.id for model in self._client.models.list().data]
         except Exception as error: