From 8fee4a07c7913f748278f899d51b2e606fedf1a0 Mon Sep 17 00:00:00 2001 From: Zhao Shenyang Date: Mon, 19 Aug 2024 14:08:58 +0800 Subject: [PATCH] tmp: for inter-service communication proposal --- service.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/service.py b/service.py index 2f41c3e..fc470ff 100644 --- a/service.py +++ b/service.py @@ -62,10 +62,18 @@ class ExchangeAssistant: llm = bentoml.depends(Llama) def __init__(self): - httpx_client, base_url = _make_httpx_client(url=Llama.url, svc=Llama) + from urllib.parse import urlparse + + base_url = "" + parsed = urlparse(Llama.url) + if parsed.scheme == "file": + base_url = "http://127.0.0.1:3000" + elif parsed.scheme == "tcp": + base_url = f"http://{parsed.netloc}" + self.client = OpenAI( base_url=f"{base_url}/v1", - http_client=httpx_client, + http_client=self.llm._sync.client, api_key="API_TOKEN_NOT_NEEDED" )