Add LM-Inline provider and unit tests

Author: christinaexyou

providers.d/inline/eval/trustyai_lmeval.yaml

@@ -0,0 +1,5 @@
+module: llama_stack_provider_lmeval.inline
+config_class: llama_stack_provider_lmeval.config.LMEvalEvalProviderConfig
+pip_packages: ["lm-eval"]
+api_dependencies: ["inference"]
+optional_api_dependencies: []

providers.d/remote/eval/trustyai_lmeval.yaml

@@ -2,6 +2,6 @@ adapter:
   adapter_type: lmeval
   pip_packages: ["kubernetes"]
   config_class: llama_stack_provider_lmeval.config.LMEvalEvalProviderConfig
-  module: llama_stack_provider_lmeval
+  module: llama_stack_provider_lmeval.remote
 api_dependencies: ["inference"]
 optional_api_dependencies: []

run-inline.yaml

@@ -0,0 +1,23 @@
+version: "2"
+image_name: trustyai-lmeval
+apis:
+  - inference
+  - eval
+providers:
+  inference:
+    - provider_id: vllm
+      provider_type: remote::vllm
+      config:
+        url: ${env.VLLM_URL:=http://localhost:8080/v1}
+        max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
+        api_token: ${env.VLLM_API_TOKEN:=fake}
+        tls_verify: ${env.VLLM_TLS_VERIFY:=false}
+  eval:
+    - provider_id: trustyai_lmeval
+      provider_type: inline::trustyai_lmeval
+      config:
+         base_url: ${env.BASE_URL:=http://localhost:8321/v1}
+         use_k8s: ${env.USE_K8S:=false}
+  # server:
+  #   port: ${env.PORT:=8321}
+external_providers_dir: ./providers.d

src/llama_stack_provider_lmeval/__init__.py

@@ -1,54 +0,0 @@
-import logging
-
-from llama_stack.apis.datatypes import Api
-from llama_stack.providers.datatypes import ProviderSpec
-
-from .config import LMEvalEvalProviderConfig
-from .lmeval import LMEval
-from .provider import get_provider_spec
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-
-async def get_adapter_impl(
-    config: LMEvalEvalProviderConfig,
-    deps: dict[Api, ProviderSpec] | None = None,
-) -> LMEval:
-    """Get an LMEval implementation from the configuration.
-
-    Args:
-        config: LMEval configuration
-        deps: Optional dependencies for testing/injection
-
-    Returns:
-        Configured LMEval implementation
-
-    Raises:
-        Exception: If configuration is invalid
-    """
-    try:
-        if deps is None:
-            deps = {}
-
-        # Extract base_url from config if available
-        base_url = None
-        if hasattr(config, "model_args") and config.model_args:
-            for arg in config.model_args:
-                if arg.get("name") == "base_url":
-                    base_url = arg.get("value")
-                    logger.debug(f"Using base_url from config: {base_url}")
-                    break
-
-        return LMEval(config=config)
-    except Exception as e:
-        raise Exception(f"Failed to create LMEval implementation: {str(e)}") from e
-
-
-__all__ = [
-    # Factory methods
-    "get_adapter_impl",
-    # Configurations
-    "LMEval",
-    "get_provider_spec",
-]

src/llama_stack_provider_lmeval/config.py

@@ -131,7 +131,6 @@ def __post_init__(self):
         if not isinstance(self.use_k8s, bool):
             raise LMEvalConfigError("use_k8s must be a boolean")
 
-
 __all__ = [
     "TLSConfig",
     "LMEvalBenchmarkConfig",

src/llama_stack_provider_lmeval/inline/__init__.py

@@ -0,0 +1,48 @@
+import logging
+from typing import Optional
+
+from llama_stack.apis.datatypes import Api
+from llama_stack.providers.datatypes import ProviderSpec
+
+from llama_stack_provider_lmeval.config import LMEvalBenchmarkConfig
+from .lmeval import LMEvalInline
+
+logger = logging.getLogger(__name__)
+
+async def get_provider_impl(
+    config: LMEvalBenchmarkConfig,
+    deps: Optional[dict[Api, ProviderSpec]] = None,
+) -> LMEvalInline:
+    """Get an inline Eval  implementation from the configuration.
+
+    Args:
+        config: LMEvalInlineBenchmarkConfig
+        deps: Optional[dict[Api, ProviderSpec]] = None
+
+    Returns:
+        Configured LMEval Inline implementation
+
+    Raises:
+        Exception: If configuration is invalid
+    """
+    try:
+        if deps is None:
+            deps = {}
+
+        # Extract base_url from config if available
+        base_url = None
+        if hasattr(config, "model_args") and config.model_args:
+            for arg in config.model_args:
+                if arg.get("name") == "base_url":
+                    base_url = arg.get("value")
+                    logger.debug(f"Using base_url from config: {base_url}")
+                    break
+
+        return LMEvalInline(config=config)
+    except Exception as e:
+        raise Exception(f"Failed to create LMEval implementation: {str(e)}") from e
+
+__all__ = [
+    "get_provider_impl",
+    "LMEvalInline",
+]