fix path params issue, make interface consistent (#78)

Author: depenglee1707

llmserve/backend/llm/pipelines/llamacpp/llamacpp_pipeline.py

@@ -242,6 +242,7 @@ def streamGenerate(self, prompt: str, **generate_kwargs) -> Generator[str, None,
         logger.info(f"stream generate_kwargs: {generate_kwargs}")
         logger.info(f"model inputs: {inputs}")
         generate_kwargs.pop('stopping_sequences', None)
+        generate_kwargs.pop('echo', None)
         if chat_completion:
             logger.info(f"chat generate_kwargs: {generate_kwargs}")
             output = self.model.create_chat_completion(messages=inputs[0], stream=True, **generate_kwargs)

llmserve/backend/server/app.py

@@ -353,19 +353,18 @@ async def metadata(self, model: str) -> Dict[str, Dict[str, Any]]:
     async def models(self) -> List[str]:
         return list(self._models.keys())
 
-    @app.post("/run/stream")
-    def streamer(self, data: dict) -> StreamingResponse:
-        logger.info(f"data: {data}")
-        logger.info(f'Got stream -> body: {data}, keys: {self._models.keys()}')
-        prompt = data.get("prompt")
-        model = data.get("model")
+    @app.post("/{model}/run/stream") 
+    def streamer(self, model: str, prompt: Union[Prompt, List[Prompt]]) -> StreamingResponse:
+        logger.info(f"url: {model}, keys: {self._models.keys()}")
+            
         modelKeys = list(self._models.keys())
         modelID = model
         for item in modelKeys:
             logger.info(f"_reverse_prefix(item): {_reverse_prefix(item)}")
             if _reverse_prefix(item) == model:
                 modelID = item
                 logger.info(f"set stream model id: {item}")
+
         logger.info(f"search stream model key: {modelID}")
         return StreamingResponse(self.streamer_generate_text(modelID, prompt), media_type="text/plain")