File tree Expand file tree Collapse file tree 3 files changed +20
-2
lines changed
Expand file tree Collapse file tree 3 files changed +20
-2
lines changed Original file line number Diff line number Diff line change @@ -110,6 +110,14 @@ def info(self) -> dict[str, Any]:
110110 """
111111 ...
112112
113+ @abstractmethod
114+ async def reset (self ) -> None :
115+ """
116+ Reset the connection object. This is useful for backends that
117+ reuse connections or have state that needs to be cleared.
118+ """
119+ ...
120+
113121 async def validate (self ):
114122 """
115123 Handle final setup and validate the backend is ready for use.
@@ -126,6 +134,8 @@ async def validate(self):
126134 ): # type: ignore[attr-defined]
127135 pass
128136
137+ await self .reset ()
138+
129139 @abstractmethod
130140 async def check_setup (self ):
131141 """
Original file line number Diff line number Diff line change @@ -157,6 +157,15 @@ def info(self) -> dict[str, Any]:
157157 "chat_completions_path" : CHAT_COMPLETIONS_PATH ,
158158 }
159159
160+ async def reset (self ) -> None :
161+ """
162+ Reset the connection object. This is useful for backends that
163+ reuse connections or have state that needs to be cleared.
164+ For this backend, it closes the async client if it exists.
165+ """
166+ if self ._async_client is not None :
167+ await self ._async_client .aclose ()
168+
160169 async def check_setup (self ):
161170 """
162171 Check if the backend is setup correctly and can be used for requests.
Original file line number Diff line number Diff line change @@ -113,8 +113,6 @@ async def run(
113113 if max_duration is not None and max_duration < 0 :
114114 raise ValueError (f"Invalid max_duration: { max_duration } " )
115115
116- # Must call before workers fork
117- await self .worker .prepare_multiprocessing ()
118116 with (
119117 multiprocessing .Manager () as manager ,
120118 ProcessPoolExecutor (
@@ -185,6 +183,7 @@ async def _start_processes(
185183 multiprocessing .Queue ,
186184 multiprocessing .Queue ,
187185 ]:
186+ await self .worker .prepare_multiprocessing ()
188187 requests_queue = manager .Queue (
189188 maxsize = scheduling_strategy .queued_requests_limit
190189 )
You can’t perform that action at this time.
0 commit comments