Improve routing-group HA sync and refresh free-tier presets

Author: successbyfailure

backend/__init__.py

@@ -1,3 +1,3 @@
 """Backend sync worker service."""
 
-__version__ = "0.6.25"
+__version__ = "0.6.28"

backend/litellm_client.py

@@ -1,5 +1,6 @@
 """LiteLLM API client for pushing models."""
 import logging
+from collections import OrderedDict
 import httpx
 
 from shared.models import ModelMetadata
@@ -220,6 +221,8 @@ async def list_routing_group_deployments(config) -> list[dict]:
                 "group": group_name,
                 "provider": _extract_tag_value(tags, "provider:") or "",
                 "model_id": _extract_tag_value(tags, "model:") or "",
+                "routing_target": _extract_tag_value(tags, "routing_target:") or "",
+                "routing_slot": _extract_tag_value(tags, "routing_slot:") or "",
                 "model_name": model.get("model_name"),
                 "model_info_id": model.get("model_info", {}).get("id"),
                 "created_by": model.get("model_info", {}).get("created_by"),
@@ -635,6 +638,86 @@ def _merge_pricing_fields(target: dict, source: dict) -> None:
             target[key] = value
 
 
+async def _set_group_fallbacks(
+    client: httpx.AsyncClient,
+    base_url: str,
+    api_key: str | None,
+    group_name: str,
+    fallback_models: list[str],
+) -> None:
+    """Configure general fallbacks for a routing group model."""
+    url = f"{base_url.rstrip('/')}/fallback"
+    headers = _make_auth_headers(api_key)
+    payload = {
+        "model": group_name,
+        "fallback_models": fallback_models,
+        "fallback_type": "general",
+    }
+    response = await client.post(url, json=payload, headers=headers, timeout=DEFAULT_TIMEOUT)
+    response.raise_for_status()
+
+
+async def _clear_group_fallbacks(
+    client: httpx.AsyncClient,
+    base_url: str,
+    api_key: str | None,
+    group_name: str,
+) -> None:
+    """Delete general fallback configuration for a routing group model."""
+    url = f"{base_url.rstrip('/')}/fallback/{group_name}"
+    headers = _make_auth_headers(api_key)
+    response = await client.delete(
+        url,
+        params={"fallback_type": "general"},
+        headers=headers,
+        timeout=DEFAULT_TIMEOUT,
+    )
+    if response.status_code == 404:
+        return
+    response.raise_for_status()
+
+
+async def delete_routing_group_from_litellm(config, group_name: str) -> dict:
+    """Delete all LiteLLM deployments/fallbacks for one routing group."""
+    if not config.litellm_base_url:
+        raise RuntimeError("LiteLLM destination not configured")
+
+    group_tag = f"routing_group:{group_name}".lower()
+    stats = {"deleted": 0, "errors": 0, "fallback_deleted": 0}
+
+    async with httpx.AsyncClient() as client:
+        litellm_models = await fetch_litellm_models(client, config.litellm_base_url, config.litellm_api_key)
+        for model in litellm_models:
+            tags = _collect_litellm_tags(model)
+            if group_tag not in tags:
+                continue
+            if model.get("model_info", {}).get("created_by") != "routing_group":
+                continue
+            model_id = model.get("model_info", {}).get("id")
+            if not model_id:
+                continue
+            try:
+                await delete_model_from_litellm(
+                    client,
+                    config.litellm_base_url,
+                    config.litellm_api_key,
+                    model_id,
+                )
+                stats["deleted"] += 1
+            except Exception as exc:
+                stats["errors"] += 1
+                logger.warning("Failed deleting routing group entry %s: %s", model_id, exc)
+
+        try:
+            await _clear_group_fallbacks(client, config.litellm_base_url, config.litellm_api_key, group_name)
+            stats["fallback_deleted"] = 1
+        except Exception as exc:
+            stats["errors"] += 1
+            logger.warning("Failed deleting fallback config for %s: %s", group_name, exc)
+
+    return stats
+
+
 async def push_routing_groups_to_litellm(session, config, group_id: int | None = None) -> dict:
     """Push routing groups to LiteLLM as model groups."""
     if not config.litellm_base_url:
@@ -650,14 +733,25 @@ async def push_routing_groups_to_litellm(session, config, group_id: int | None =
         groups = [group] if group else []
 
     groups = [g for g in groups if g is not None]
-    stats = {"groups": len(groups), "added": 0, "deleted": 0, "missing_models": 0, "errors": 0}
+    stats = {
+        "groups": len(groups),
+        "added": 0,
+        "deleted": 0,
+        "missing_models": 0,
+        "errors": 0,
+        "fallbacks_updated": 0,
+        "fallbacks_deleted": 0,
+        "targets_skipped_disabled": 0,
+    }
 
     async with httpx.AsyncClient() as client:
         litellm_models = await fetch_litellm_models(client, config.litellm_base_url, config.litellm_api_key)
 
         for group in groups:
             group_tag = f"routing_group:{group.name}"
             group_tag_lower = group_tag.lower()
+            fallback_models: list[str] = []
+            seen_fallback_models: OrderedDict[str, bool] = OrderedDict()
 
             for m in litellm_models:
                 tags = m.get("litellm_params", {}).get("tags", [])
@@ -684,6 +778,9 @@ async def push_routing_groups_to_litellm(session, config, group_id: int | None =
                     logger.warning("Failed deleting routing group entry %s: %s", model_id, exc)
 
             for target in sorted(group.targets, key=lambda t: (t.priority, t.id)):
+                if not target.enabled:
+                    stats["targets_skipped_disabled"] += 1
+                    continue
                 provider = target.provider or await get_provider_by_id(session, target.provider_id)
                 if not provider:
                     stats["missing_models"] += 1
@@ -692,29 +789,66 @@ async def push_routing_groups_to_litellm(session, config, group_id: int | None =
                 if not model:
                     stats["missing_models"] += 1
                     continue
+
+                fallback_name = model.get_display_name(apply_prefix=True)
+                if fallback_name and fallback_name not in seen_fallback_models:
+                    seen_fallback_models[fallback_name] = True
+                    fallback_models.append(fallback_name)
+
+                slot_count = max(1, int(target.weight or 1))
+                routing_target_tag = f"routing_target:{provider.id}:{model.model_id}"
+                for slot in range(1, slot_count + 1):
+                    extra_tags = [group_tag, routing_target_tag, f"routing_slot:{slot}"]
+                    try:
+                        await push_model_to_litellm(
+                            client,
+                            config.litellm_base_url,
+                            config.litellm_api_key,
+                            provider,
+                            model,
+                            config=config,
+                            session=session,
+                            model_name_override=group.name,
+                            extra_tags=extra_tags,
+                            created_by="routing_group",
+                            strip_unique_id=True,
+                        )
+                        stats["added"] += 1
+                    except Exception as exc:
+                        stats["errors"] += 1
+                        logger.warning(
+                            "Failed pushing routing target %s/%s (slot=%s) for group %s: %s",
+                            provider.name,
+                            model.model_id,
+                            slot,
+                            group.name,
+                            exc,
+                        )
+
+            if fallback_models:
                 try:
-                    await push_model_to_litellm(
+                    await _set_group_fallbacks(
                         client,
                         config.litellm_base_url,
                         config.litellm_api_key,
-                        provider,
-                        model,
-                        config=config,
-                        session=session,
-                        model_name_override=group.name,
-                        extra_tags=[group_tag],
-                        created_by="routing_group",
-                        strip_unique_id=True,
+                        group.name,
+                        fallback_models,
                     )
-                    stats["added"] += 1
+                    stats["fallbacks_updated"] += 1
                 except Exception as exc:
                     stats["errors"] += 1
-                    logger.warning(
-                        "Failed pushing routing target %s/%s for group %s: %s",
-                        provider.name,
-                        model.model_id,
+                    logger.warning("Failed updating fallback config for group %s: %s", group.name, exc)
+            else:
+                try:
+                    await _clear_group_fallbacks(
+                        client,
+                        config.litellm_base_url,
+                        config.litellm_api_key,
                         group.name,
-                        exc,
                     )
+                    stats["fallbacks_deleted"] += 1
+                except Exception as exc:
+                    stats["errors"] += 1
+                    logger.warning("Failed clearing fallback config for empty group %s: %s", group.name, exc)
 
     return stats

frontend/__init__.py

@@ -1,3 +1,3 @@
 """Frontend API and UI service."""
 
-__version__ = "0.6.25"
+__version__ = "0.6.28"

frontend/routes/routing_groups.py

@@ -1,6 +1,8 @@
 """Routing group management API routes."""
 from __future__ import annotations
 
+from collections import Counter
+
 from fastapi import APIRouter, Depends, HTTPException, Query
 from pydantic import BaseModel, Field
 from sqlalchemy import select
@@ -9,7 +11,11 @@
 
 from shared.database import get_session
 from shared.crud import get_config
-from backend.litellm_client import push_routing_groups_to_litellm, list_routing_group_deployments
+from backend.litellm_client import (
+    push_routing_groups_to_litellm,
+    list_routing_group_deployments,
+    delete_routing_group_from_litellm,
+)
 from shared.crud import (
     get_routing_groups,
     get_routing_group,
@@ -123,7 +129,15 @@ async def create_group(
     group = await get_routing_group(session, group.id)
     if not group:
         raise HTTPException(404, "Routing group not found after create")
-    return _group_to_dict(group)
+    response = _group_to_dict(group)
+
+    config = await get_config(session)
+    if config.litellm_base_url:
+        response["sync"] = await push_routing_groups_to_litellm(session, config, group_id=group.id)
+    else:
+        response["sync"] = {"skipped": "LiteLLM destination not configured"}
+
+    return response
 
 
 @router.get("/candidates")
@@ -174,42 +188,57 @@ async def routing_group_status(session: AsyncSession = Depends(get_session)) ->
 
     for group in groups:
         db_targets = []
-        db_keys = set()
+        db_counts: Counter[str] = Counter()
         for target in sorted(group.targets, key=lambda t: (t.priority, t.id)):
             provider_name = target.provider.name if target.provider else None
+            weight = max(1, int(target.weight or 1))
             db_targets.append(
                 {
                     "provider_name": provider_name,
                     "model_id": target.model_id,
+                    "weight": weight,
                     "enabled": target.enabled,
                 }
             )
             if target.enabled:
-                db_keys.add(_target_key(provider_name, target.model_id))
+                db_counts[_target_key(provider_name, target.model_id)] += weight
 
         litellm_targets = litellm_by_group.get(group.name, [])
-        litellm_keys = {
+        litellm_counts: Counter[str] = Counter(
             _target_key(entry.get("provider"), entry.get("model_id"))
             for entry in litellm_targets
             if entry.get("provider") and entry.get("model_id")
-        }
+        )
 
         missing_in_litellm = [
-            target for target in db_targets
-            if target["enabled"] and _target_key(target["provider_name"], target["model_id"]) not in litellm_keys
-        ]
-        extra_in_litellm = [
-            entry for entry in litellm_targets
-            if _target_key(entry.get("provider"), entry.get("model_id")) not in db_keys
+            {
+                "provider_name": item["provider_name"],
+                "model_id": item["model_id"],
+                "expected": item["weight"],
+                "actual": litellm_counts.get(_target_key(item["provider_name"], item["model_id"]), 0),
+            }
+            for item in db_targets
+            if item["enabled"]
+            and litellm_counts.get(_target_key(item["provider_name"], item["model_id"]), 0) < item["weight"]
         ]
+        extra_in_litellm = []
+        for entry in litellm_targets:
+            key = _target_key(entry.get("provider"), entry.get("model_id"))
+            expected = db_counts.get(key, 0)
+            if expected <= 0:
+                extra_in_litellm.append(entry)
+                continue
+            if litellm_counts[key] > expected:
+                extra_in_litellm.append(entry)
+                litellm_counts[key] -= 1
 
         response_groups.append(
             {
                 "id": group.id,
                 "name": group.name,
                 "description": group.description,
                 "db_targets": db_targets,
-                "db_count": len([t for t in db_targets if t["enabled"]]),
+                "db_count": sum(db_counts.values()),
                 "litellm_count": len(litellm_targets),
                 "litellm_targets": litellm_targets,
                 "missing_in_litellm": missing_in_litellm,
@@ -284,7 +313,13 @@ async def update_group(
     except IntegrityError as exc:
         raise HTTPException(400, "Invalid routing group payload") from exc
 
-    return _group_to_dict(group)
+    response = _group_to_dict(group)
+    config = await get_config(session)
+    if config.litellm_base_url:
+        response["sync"] = await push_routing_groups_to_litellm(session, config, group_id=group.id)
+    else:
+        response["sync"] = {"skipped": "LiteLLM destination not configured"}
+    return response
 
 
 @router.delete("/{group_id}")
@@ -293,5 +328,9 @@ async def remove_group(group_id: int, session: AsyncSession = Depends(get_sessio
     group = await get_routing_group(session, group_id, include_children=False)
     if not group:
         raise HTTPException(404, "Routing group not found")
+    config = await get_config(session)
+    cleanup = None
+    if config.litellm_base_url:
+        cleanup = await delete_routing_group_from_litellm(config, group.name)
     await delete_routing_group(session, group)
-    return {"status": "ok"}
+    return {"status": "ok", "cleanup": cleanup}

frontend/templates/routing_groups.html

@@ -473,11 +473,11 @@ <h4>${group.name}</h4>
       return;
     }
 
-    summary.textContent = `DB targets (enabled): ${status.db_count} · LiteLLM deployments: ${status.litellm_count}`;
+    summary.textContent = `Expected deployments: ${status.db_count} · LiteLLM deployments: ${status.litellm_count}`;
 
     if (status.missing_in_litellm.length) {
       const missingList = status.missing_in_litellm
-        .map((item) => `${item.provider_name || "unknown"} / ${item.model_id}`)
+        .map((item) => `${item.provider_name || "unknown"} / ${item.model_id} (${item.actual}/${item.expected})`)
         .join(", ");
       missingEl.textContent = `Missing in LiteLLM: ${missingList}`;
     } else {

proxy/__init__.py

@@ -1 +1 @@
-__version__ = "0.6.25"
+__version__ = "0.6.28"

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-companion"
-version = "0.6.25"
+version = "0.6.28"
 description = "Synchronize models from Ollama or OpenAI-compatible endpoints into LiteLLM"
 authors = [
   {name = "LiteLLM Companion Authors", email = "dev@example.com"}

shared/__init__.py

@@ -1,3 +1,3 @@
 """Shared code between backend and frontend services."""
 
-__version__ = "0.6.25"
+__version__ = "0.6.28"