Surface community config health via public API (#221)

* Surface community config health via public API

Downgrade noisy API key warning from error to debug level
in discover_assistants() so it stops spamming CLI users.

Extract compute_community_health() helper in health.py and
add config_health to GET /{community_id}/metrics/public and
status field to GET /{community_id}/ config response.

Warnings clearly communicate that using the shared platform
key is for demonstration only and not sustainable.

Closes #220

* Address PR review: fix error handling and security

- Move compute_community_health() call inside try/except in
  get_communities_health() to restore error containment
- Add try/except around health computation in config and
  public metrics endpoints so failures don't crash them
- Sanitize warnings for public endpoint: strip env var names
  to prevent information disclosure
- Change logger.debug to logger.warning in discover_assistants
  so operators see missing key warnings at startup
- Move inline imports to top-level (no circular import risk)
- Remove redundant "error" key from fallback dict

* Fix CI: initialize temp metrics DB in health status tests

The public metrics tests need a metrics DB to avoid 503.
Use the same tmp_path + patch pattern as test_metrics_public.

Author: neuromechanist

src/api/routers/community.py

@@ -25,6 +25,7 @@
 
 from src.agents.base import DEFAULT_MAX_CONVERSATION_TOKENS
 from src.api.config import get_settings
+from src.api.routers.health import compute_community_health
 from src.api.security import AuthScope, RequireAuth, RequireScopedAuth
 from src.assistants import registry
 from src.assistants.community import CommunityAssistant
@@ -188,6 +189,7 @@ class CommunityConfigResponse(BaseModel):
     widget: WidgetConfigResponse = Field(
         ..., description="Widget display configuration (title, placeholder, etc.)"
     )
+    status: str = Field(..., description="Health status: healthy, degraded, or error")
 
 
 # ---------------------------------------------------------------------------
@@ -1121,13 +1123,22 @@ async def get_community_config() -> CommunityConfigResponse:
             info.community_config.widget if info.community_config else None
         ) or WidgetConfig()
 
+        # Compute lightweight health status for public display
+        health_status = "error"
+        if info.community_config:
+            try:
+                health_status = compute_community_health(info.community_config)["status"]
+            except Exception:
+                logger.exception("Failed to compute health for community %s", info.id)
+
         return CommunityConfigResponse(
             id=info.id,
             name=info.name,
             description=info.description,
             default_model=default_model,
             default_model_provider=default_provider,
             widget=WidgetConfigResponse(**widget_cfg.resolve(info.name)),
+            status=health_status,
         )
 
     # -----------------------------------------------------------------------
@@ -1224,19 +1235,50 @@ async def community_quality_summary(auth: RequireScopedAuth) -> dict[str, Any]:
     async def community_metrics_public() -> dict[str, Any]:
         """Get public metrics summary for this community.
 
-        Returns request counts, error rate, and top tools.
+        Returns request counts, error rate, top tools, and config health.
         No tokens, costs, or model information exposed.
         """
         try:
             with metrics_connection() as conn:
-                return get_public_community_summary(community_id, conn)
+                result = get_public_community_summary(community_id, conn)
         except sqlite3.Error:
             logger.exception("Failed to query public metrics for community %s", community_id)
             raise HTTPException(
                 status_code=503,
                 detail="Metrics database is temporarily unavailable.",
             )
 
+        # Add config health alongside usage metrics
+        fallback_health: dict[str, Any] = {
+            "status": "error",
+            "api_key": "missing",
+            "documents": 0,
+            "warnings": ["Community configuration not found"],
+        }
+        if info.community_config:
+            try:
+                health = compute_community_health(info.community_config)
+                # Sanitize warnings for public endpoint: strip env var names
+                public_warnings = [w for w in health["warnings"] if "Environment variable" not in w]
+                if health["api_key"] == "missing" and not public_warnings:
+                    public_warnings = [
+                        "API key not configured; using shared platform key. "
+                        "This is for demonstration only and is not sustainable."
+                    ]
+                result["config_health"] = {
+                    "status": health["status"],
+                    "api_key": health["api_key"],
+                    "documents": health["documents"],
+                    "warnings": public_warnings,
+                }
+            except Exception:
+                logger.exception("Failed to compute health for community %s", community_id)
+                result["config_health"] = fallback_health
+        else:
+            result["config_health"] = fallback_health
+
+        return result
+
     @router.get("/metrics/public/usage")
     async def community_usage_public(
         period: str = Query(

src/api/routers/health.py

@@ -8,11 +8,60 @@
 
 from src.api.security import RequireAuth
 from src.assistants import registry
+from src.core.config.community import CommunityConfig
 
 router = APIRouter(prefix="/health", tags=["health"])
 logger = logging.getLogger(__name__)
 
 
+def compute_community_health(config: CommunityConfig) -> dict[str, Any]:
+    """Compute health status for a single community config.
+
+    Returns:
+        Dict with status, api_key, cors_origins, documents, sync_age_hours, warnings.
+    """
+    warnings: list[str] = []
+
+    # API key status
+    api_key_env_var = config.openrouter_api_key_env_var
+    if api_key_env_var:
+        if os.getenv(api_key_env_var):
+            api_key_status = "configured"
+        else:
+            api_key_status = "missing"
+            warnings.append(
+                f"Environment variable '{api_key_env_var}' not set; "
+                "using shared OSA platform key. This is for demonstration only "
+                "and is not sustainable. Requires a dedicated API key and active maintainer."
+            )
+    else:
+        api_key_status = "using_platform"
+
+    # Documentation sources
+    doc_count = len(config.documentation) if config.documentation else 0
+    if doc_count == 0:
+        warnings.append("No documentation sources configured")
+
+    # CORS origins
+    cors_count = len(config.cors_origins) if config.cors_origins else 0
+
+    # Determine overall status
+    status = "healthy"
+    if doc_count == 0 or api_key_status == "missing":
+        status = "error"
+    elif api_key_status == "using_platform":
+        status = "degraded"
+
+    return {
+        "status": status,
+        "api_key": api_key_status,
+        "cors_origins": cors_count,
+        "documents": doc_count,
+        "sync_age_hours": None,  # TODO: Add sync tracking in future iteration
+        "warnings": warnings,
+    }
+
+
 @router.get("/communities")
 def get_communities_health(_auth: RequireAuth) -> dict[str, Any]:
     """Get health status for all communities.
@@ -23,6 +72,7 @@ def get_communities_health(_auth: RequireAuth) -> dict[str, Any]:
     - cors_origins: number of CORS origins configured
     - documents: number of documentation sources
     - sync_age_hours: hours since last sync (if applicable)
+    - warnings: list of configuration issues
 
     Returns:
         Dictionary mapping community IDs to their health status.
@@ -60,8 +110,10 @@ def get_communities_health(_auth: RequireAuth) -> dict[str, Any]:
                     "cors_origins": 0,
                     "documents": 0,
                     "sync_age_hours": None,
+                    "warnings": ["Community configuration not found"],
                 }
                 continue
+            communities_health[community_id] = compute_community_health(config)
         except (AttributeError, KeyError, TypeError) as e:
             logger.error(
                 "Failed to process community health for %s: %s",
@@ -81,51 +133,12 @@ def get_communities_health(_auth: RequireAuth) -> dict[str, Any]:
             )
             communities_health[fallback_id] = {
                 "status": "error",
-                "error": f"Failed to process: {type(e).__name__}",
                 "api_key": "unknown",
                 "cors_origins": 0,
                 "documents": 0,
                 "sync_age_hours": None,
+                "warnings": [f"Failed to process: {type(e).__name__}"],
             }
             continue
 
-        # Determine API key status
-        api_key_env_var = getattr(config, "openrouter_api_key_env_var", None)
-        if api_key_env_var:
-            # Check if env var is actually set
-            api_key_status = "configured" if os.getenv(api_key_env_var) else "missing"
-        else:
-            api_key_status = "using_platform"
-
-        # Count documentation sources
-        documentation = getattr(config, "documentation", None)
-        doc_count = len(documentation) if documentation else 0
-
-        # Count CORS origins
-        cors_origins = getattr(config, "cors_origins", None)
-        cors_count = len(cors_origins) if cors_origins else 0
-
-        # Sync age is not tracked yet, set to None
-        # TODO: Add sync tracking in future iteration
-        sync_age_hours = None
-
-        # Determine overall status
-        # - error: critical issues (no docs, missing configured API key)
-        # - degraded: warnings (using platform key)
-        # - healthy: all good
-        status = "healthy"
-
-        if doc_count == 0 or api_key_status == "missing":
-            status = "error"
-        elif api_key_status == "using_platform":
-            status = "degraded"
-
-        communities_health[community_id] = {
-            "status": status,
-            "api_key": api_key_status,
-            "cors_origins": cors_count,
-            "documents": doc_count,
-            "sync_age_hours": sync_age_hours,
-        }
-
     return communities_health

src/assistants/__init__.py

@@ -79,21 +79,15 @@ def discover_assistants() -> list[str]:
             discovered.append(config.id)
             logger.info("Discovered assistant: %s from %s", config.id, config_path)
 
-            # Validate API key env var is set if configured
+            # Warn once at startup; detailed health surfaced via API endpoints.
+            # See: https://github.com/OpenScience-Collective/osa/issues/220
             if config.openrouter_api_key_env_var and not os.getenv(
                 config.openrouter_api_key_env_var
             ):
-                logger.error(
-                    "Community '%s' configured to use env var '%s' but it is not set. "
-                    "This community will fall back to the platform API key, which may incur unexpected costs. "
-                    "Set the environment variable or remove 'openrouter_api_key_env_var' from config.yaml",
+                logger.warning(
+                    "Community '%s': env var '%s' not set, will use platform key",
                     config.id,
                     config.openrouter_api_key_env_var,
-                    extra={
-                        "community_id": config.id,
-                        "env_var": config.openrouter_api_key_env_var,
-                        "env_var_missing": True,
-                    },
                 )
         except KeyboardInterrupt:
             # Never catch user interruption

tests/test_api/test_community_router.py

@@ -5,8 +5,11 @@
 - Dynamic endpoint registration
 - Session isolation between communities
 - Backward compatibility with HED endpoints
+- Public health status in config and metrics endpoints
 """
 
+import os
+
 import pytest
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
@@ -346,3 +349,92 @@ def test_session_delete_endpoint_exists(self) -> None:
         else:
             # Auth required
             assert response.status_code in (401, 403)
+
+
+class TestCommunityConfigHealthStatus:
+    """Tests for health status in community config and public metrics."""
+
+    @pytest.fixture
+    def client(self, tmp_path) -> TestClient:
+        """Create a test client with auth disabled and metrics DB initialized."""
+        os.environ["REQUIRE_API_AUTH"] = "false"
+        from src.api.config import get_settings
+
+        get_settings.cache_clear()
+
+        # Initialize a temp metrics DB so /metrics/public doesn't 503
+        from unittest.mock import patch
+
+        from src.metrics.db import init_metrics_db
+
+        db_path = tmp_path / "metrics.db"
+        init_metrics_db(db_path)
+
+        from src.api.main import app
+
+        with patch("src.metrics.db.get_metrics_db_path", return_value=db_path):
+            yield TestClient(app)
+
+    def test_config_response_includes_status(self, client: TestClient) -> None:
+        """GET /{community_id}/ should include a status field."""
+        response = client.get("/hed/")
+        assert response.status_code == 200
+
+        data = response.json()
+        assert "status" in data
+        assert data["status"] in ["healthy", "degraded", "error"]
+
+    def test_config_status_does_not_leak_details(self, client: TestClient) -> None:
+        """Public config should not expose api_key details or warnings."""
+        response = client.get("/hed/")
+        data = response.json()
+
+        assert "warnings" not in data
+        assert "api_key" not in data
+        assert "config_health" not in data
+
+    def test_public_metrics_includes_config_health(self, client: TestClient) -> None:
+        """GET /{community_id}/metrics/public should include config_health."""
+        response = client.get("/hed/metrics/public")
+        assert response.status_code == 200
+
+        data = response.json()
+        assert "config_health" in data
+
+        health = data["config_health"]
+        assert "status" in health
+        assert health["status"] in ["healthy", "degraded", "error"]
+        assert "api_key" in health
+        assert health["api_key"] in ["configured", "using_platform", "missing"]
+        assert "documents" in health
+        assert isinstance(health["documents"], int)
+        assert "warnings" in health
+        assert isinstance(health["warnings"], list)
+
+    def test_public_metrics_config_health_has_warnings_for_missing_key(
+        self, client: TestClient
+    ) -> None:
+        """config_health should include warnings when API key env var is not set."""
+        from src.assistants import registry
+
+        # Find a community with openrouter_api_key_env_var
+        for assistant in registry.list_all():
+            config = assistant.community_config
+            if config and config.openrouter_api_key_env_var:
+                env_var = config.openrouter_api_key_env_var
+                original = os.environ.pop(env_var, None)
+                try:
+                    response = client.get(f"/{assistant.id}/metrics/public")
+                    assert response.status_code == 200
+                    health = response.json()["config_health"]
+                    assert health["api_key"] == "missing"
+                    assert len(health["warnings"]) > 0
+                    assert any("not sustainable" in w for w in health["warnings"])
+                    # Env var names must not leak to public endpoint
+                    assert not any(env_var in w for w in health["warnings"])
+                finally:
+                    if original is not None:
+                        os.environ[env_var] = original
+                return
+
+        pytest.skip("No community with openrouter_api_key_env_var configured")