Infra improvements

lm_proxy/__main__.py

@@ -1,4 +1,5 @@
 """Provides the CLI entry point when the package is executed as a Python module."""
+
 from .app import cli_app
 
 

lm_proxy/api_key_check/__init__.py

@@ -1,4 +1,5 @@
 """Collection of built-in API-key checkers for usage in the configuration."""
+
 from .in_config import check_api_key_in_config
 from .with_request import CheckAPIKeyWithRequest
 from .allow_all import AllowAll

lm_proxy/api_key_check/allow_all.py

@@ -4,6 +4,7 @@
 This module provides a simple authentication strategy for development or testing
 environments where all API keys should be accepted without validation.
 """
+
 from typing import Optional
 from dataclasses import dataclass
 
@@ -25,10 +26,7 @@ class AllowAll:
     group: str = "default"
     capture_api_key: bool = True
 
-    def __call__(
-        self,
-        api_key: Optional[str]
-    ) -> tuple[str, dict[str, Optional[str]]]:
+    def __call__(self, api_key: Optional[str]) -> tuple[str, dict[str, Optional[str]]]:
         """
         Validate an API key (accepts all keys without verification).
 

lm_proxy/api_key_check/in_config.py

@@ -5,6 +5,7 @@
 For using this function,
 set "api_key_check" configuration value to "lm_proxy.api_key_check.check_api_key_in_config".
 """
+
 from typing import Optional
 from ..bootstrap import env
 

lm_proxy/api_key_check/with_request.py

@@ -1,6 +1,7 @@
 """
 API key check implementation using HTTP requests.
 """
+
 from typing import Optional
 from dataclasses import dataclass, field
 import requests
@@ -13,6 +14,7 @@ class CheckAPIKeyWithRequest:  # pylint: disable=too-many-instance-attributes
     """
     Validates a Client API key by making an HTTP request to a specified URL.
     """
+
     url: str = field()
     method: str = field(default="get")
     headers: dict = field(default_factory=dict)
@@ -45,10 +47,7 @@ def check_func(api_key: str) -> Optional[tuple[str, dict]]:
                     for k, v in self.headers.items()
                 }
                 response = requests.request(
-                    method=self.method,
-                    url=url,
-                    headers=headers,
-                    timeout=self.timeout
+                    method=self.method, url=url, headers=headers, timeout=self.timeout
                 )
                 response.raise_for_status()
                 group = self.default_group

lm_proxy/app.py

@@ -1,6 +1,7 @@
 """
 LM-Proxy Application Entrypoint
 """
+
 import logging
 from typing import Optional
 from fastapi import FastAPI
@@ -19,9 +20,7 @@
 @cli_app.callback(invoke_without_command=True)
 def run_server(
     config: Optional[str] = typer.Option(None, help="Path to the configuration file"),
-    debug: Optional[bool] = typer.Option(
-        None, help="Enable debug mode (more verbose logging)"
-    ),
+    debug: Optional[bool] = typer.Option(None, help="Enable debug mode (more verbose logging)"),
     env_file: Optional[str] = typer.Option(
         ".env",
         "--env",
@@ -55,9 +54,7 @@ def web_app():
     """
     Entrypoint for ASGI server
     """
-    app = FastAPI(
-        title="LM-Proxy", description="OpenAI-compatible proxy server for LLM inference"
-    )
+    app = FastAPI(title="LM-Proxy", description="OpenAI-compatible proxy server for LLM inference")
     OpenAIHTTPException.register(app)
     app.add_api_route(
         path=f"{env.config.api_prefix}/chat/completions",

lm_proxy/base_types.py

@@ -1,4 +1,5 @@
 """Base types used in LM-Proxy."""
+
 import uuid
 from dataclasses import dataclass, field
 from datetime import datetime
@@ -17,6 +18,7 @@ class ChatCompletionRequest(BaseModel):
     """
     Request model for chat/completions endpoint.
     """
+
     model: str
     messages: List[mc.Msg | dict]
     # | dict --> support of messages with lists of dicts
@@ -58,6 +60,7 @@ class RequestContext:  # pylint: disable=too-many-instance-attributes
     """
     Stores information about a single LLM request/response cycle for usage in middleware.
     """
+
     id: Optional[str] = field(default_factory=lambda: str(uuid.uuid4()))
     request: Optional[ChatCompletionRequest] = field(default=None)
     http_request: Optional[Request] = field(default=None)
@@ -83,7 +86,4 @@ def to_dict(self) -> dict:
         return data
 
 
-THandler = Callable[
-    [RequestContext],
-    Union[Awaitable[None], None]
-]
+THandler = Callable[[RequestContext], Union[Awaitable[None], None]]

lm_proxy/bootstrap.py

@@ -44,6 +44,7 @@ def format(self, record):
 
 class Env:
     """Runtime environment singleton."""
+
     config: Config
     connections: dict[str, mc.types.LLMAsyncFunctionType]
     debug: bool
@@ -66,9 +67,7 @@ def init(config: Config | str | PathLike, debug: bool = False):
             if isinstance(config, (str, PathLike)):
                 config = Config.load(config)
             else:
-                raise ValueError(
-                    "config must be a path (str or PathLike) or Config instance"
-                )
+                raise ValueError("config must be a path (str or PathLike) or Config instance")
         env.config = config
 
         env._init_components()
@@ -84,18 +83,12 @@ def init(config: Config | str | PathLike, debug: bool = False):
                 if inspect.iscoroutinefunction(conn_config):
                     env.connections[conn_name] = conn_config
                 elif isinstance(conn_config, str):
-                    env.connections[conn_name] = resolve_instance_or_callable(
-                        conn_config
-                    )
+                    env.connections[conn_name] = resolve_instance_or_callable(conn_config)
                 else:
-                    mc.configure(
-                        **conn_config, EMBEDDING_DB_TYPE=mc.EmbeddingDbType.NONE
-                    )
+                    mc.configure(**conn_config, EMBEDDING_DB_TYPE=mc.EmbeddingDbType.NONE)
                     env.connections[conn_name] = mc.env().llm_async_function
             except mc.LLMConfigError as e:
-                raise ValueError(
-                    f"Error in configuration for connection '{conn_name}': {e}"
-                ) from e
+                raise ValueError(f"Error in configuration for connection '{conn_name}': {e}") from e
 
         logging.info("Done initializing %d connections.", len(env.connections))
 

lm_proxy/config.py

@@ -30,6 +30,7 @@ class ModelListingMode(StrEnum):
 
 class Group(BaseModel):
     """User group configuration."""
+
     api_keys: list[str] = Field(default_factory=list)
     allowed_connections: str = Field(default="*")  # Comma-separated list or "*"
 

lm_proxy/config_loaders/__init__.py

@@ -1,4 +1,5 @@
 """Built-in configuration loaders for different file formats."""
+
 from .python import load_python_config
 from .toml import load_toml_config
 from .yaml import load_yaml_config

lm_proxy/config_loaders/json.py

@@ -1,4 +1,5 @@
 """JSON configuration loader."""
+
 import json
 
 

lm_proxy/config_loaders/python.py

@@ -1,4 +1,5 @@
 """Loader for Python configuration files."""
+
 import importlib.util
 from ..config import Config
 

lm_proxy/config_loaders/toml.py

@@ -1,4 +1,5 @@
 """TOML configuration loader."""
+
 import tomllib
 
 

lm_proxy/core.py

@@ -46,9 +46,7 @@
     return connection_name, model_part
 
 
-def resolve_connection_and_model(
-    config: Config, external_model: str
-) -> tuple[str, str]:
+def resolve_connection_and_model(config: Config, external_model: str) -> tuple[str, str]:
     """
     Resolves the connection name and model name based on routing rules.
     Args:
@@ -150,9 +148,7 @@
     """
     if not api_key:
         return None
-    return hashlib.md5(
-        (api_key + env.config.encryption_key).encode("utf-8")
-    ).hexdigest()
+    return hashlib.md5((api_key + env.config.encryption_key).encode("utf-8")).hexdigest()
 
 
 def fail_if_service_disabled():
@@ -207,9 +203,7 @@
     return group, api_key, user_info
 
 
-async def chat_completions(
-    request: ChatCompletionRequest, raw_request: Request
-) -> Response:
+async def chat_completions(request: ChatCompletionRequest, raw_request: Request) -> Response:
     """
     Endpoint for chat completions that mimics OpenAI's API structure.
     """

lm_proxy/errors.py

@@ -6,6 +6,7 @@ class OpenAIHTTPException(Exception):
     """
     OpenAI API-compatible exception.
     """
+
     # HTTPException from FastAPI is not used directly to provide error response format
     # compatible with OpenAI API.
 

lm_proxy/handlers/__init__.py

@@ -1,7 +1,4 @@
 from .rate_limiter import RateLimiter
 from .forward_http_headers import HTTPHeadersForwarder
 
-__all__ = [
-    'RateLimiter',
-    'HTTPHeadersForwarder'
-]
+__all__ = ["RateLimiter", "HTTPHeadersForwarder"]

lm_proxy/handlers/forward_http_headers.py

@@ -1,6 +1,7 @@
 """
 HTTP headers forwarder LM-Proxy.
 """
+
 from dataclasses import dataclass, field
 
 from lm_proxy.base_types import RequestContext

lm_proxy/handlers/rate_limiter.py

@@ -3,6 +3,7 @@
 
 Provides sliding window rate limiting per API key / IP address / connection / user group / global.
 """
+
 import threading
 import time
 from dataclasses import dataclass, field
@@ -33,6 +34,7 @@ class RateLimiter:
         per: Scope for rate limiting
             ("api_key", "connection", "group", "ip", "global").
     """
+
     max_requests: int = 60
     window_seconds: float = 60.0
     per: RateLimitScope = RateLimitScope.API_KEY
@@ -68,10 +70,7 @@ async def __call__(self, ctx: RequestContext) -> None:
 
         with self._lock:
             if len(self._buckets) > self.max_buckets:
-                self._buckets = {
-                    k: v for k, v in self._buckets.items()
-                    if v and v[-1] > cutoff
-                }
+                self._buckets = {k: v for k, v in self._buckets.items() if v and v[-1] > cutoff}
 
             timestamps = [t for t in self._buckets.get(key, []) if t > cutoff]
 

lm_proxy/loggers.py

@@ -1,4 +1,5 @@
 """LLM Request logging."""
+
 import abc
 import json
 import os
@@ -11,13 +12,15 @@
 
 class AbstractLogEntryTransformer(abc.ABC):  # pylint: disable=too-few-public-methods
     """Transform RequestContext into a dictionary of logged attributes."""
+
     @abc.abstractmethod
     def __call__(self, request_context: RequestContext) -> dict:
         raise NotImplementedError()
 
 
 class AbstractLogWriter(abc.ABC):  # pylint: disable=too-few-public-methods
     """Writes the logged data to a destination."""
+
     @abc.abstractmethod
     def __call__(self, logged_data: dict):
         raise NotImplementedError()
@@ -29,6 +32,7 @@ class LogEntryTransformer(AbstractLogEntryTransformer):  # pylint: disable=too-f
     The mapping is provided as keyword arguments, where keys are the names of the
     logged attributes, and values are the paths to the attributes in RequestContext.
     """
+
     def __init__(self, **kwargs):
         self.mapping = kwargs
 
@@ -42,6 +46,7 @@ def __call__(self, request_context: RequestContext) -> dict:
 @dataclass
 class BaseLogger:
     """Base LLM request logger."""
+
     log_writer: AbstractLogWriter | str | dict
     entry_transformer: AbstractLogEntryTransformer | str | dict = field(default=None)
 
@@ -69,6 +74,7 @@ def __call__(self, request_context: RequestContext):
 @dataclass
 class JsonLogWriter(AbstractLogWriter):
     """Writes logged data to a JSON file."""
+
     file_name: str
 
     def __post_init__(self):

lm_proxy/models_endpoint.py

@@ -24,21 +24,18 @@ async def models(request: Request) -> JSONResponse:
             is_model_name = not ("*" in model_pattern or "?" in model_pattern)
             if not is_model_name:
                 if env.config.model_listing_mode != ModelListingMode.AS_IS:
-                    if (
-                        env.config.model_listing_mode
-                        == ModelListingMode.IGNORE_WILDCARDS
-                    ):
+                    if env.config.model_listing_mode == ModelListingMode.IGNORE_WILDCARDS:
                         continue
                     raise NotImplementedError(
                         f"'{env.config.model_listing_mode}' model listing mode "
                         f"is not implemented yet"
                     )
             model_data = {
-                    "id": model_pattern,
-                    "object": "model",
-                    "created": 0,
-                    "owned_by": connection_name,
-                }
+                "id": model_pattern,
+                "object": "model",
+                "created": 0,
+                "owned_by": connection_name,
+            }
 
             if aux_info := env.config.model_info.get(model_pattern):
                 model_data.update(aux_info)

multi-build.py

@@ -26,16 +26,17 @@ def replace_name(old_names: list[str], new_names: list[str], files: list[str] =
             p = Path(path)
             p.write_text(
                 re.sub(
-                    fr'(?<![\\/\w]){old_name}\b',
+                    rf"(?<![\\/\w]){old_name}\b",
                     new_name,
                     p.read_text(encoding="utf-8"),
-                    flags=re.M
-                ), encoding="utf-8"
+                    flags=re.M,
+                ),
+                encoding="utf-8",
             )
 
 
 prev = NAMES[0]
-for nxt in NAMES[1:]+[NAMES[0]]:
+for nxt in NAMES[1:] + [NAMES[0]]:
     print(f"Building for project name: {nxt[0]}...")
     replace_name(prev, nxt)
     subprocess.run(["poetry", "build"], check=True)

pyproject.toml

@@ -88,3 +88,5 @@ asyncio_mode = "auto"
 testpaths = [
     "tests",
 ]
+[tool.black]
+line-length = 100