Add LM-Eval inline provider and unit tests

Author: christinaexyou

providers.d/inline/eval/trustyai_lmeval.yaml

@@ -0,0 +1,5 @@
+module: llama_stack_provider_lmeval.inline
+config_class: llama_stack_provider_lmeval.config.LMEvalEvalProviderConfig
+pip_packages: ["lm-eval"]
+api_dependencies: ["inference"]
+optional_api_dependencies: []

providers.d/remote/eval/trustyai_lmeval.yaml

@@ -2,6 +2,6 @@ adapter:
   adapter_type: lmeval
   pip_packages: ["kubernetes"]
   config_class: llama_stack_provider_lmeval.config.LMEvalEvalProviderConfig
-  module: llama_stack_provider_lmeval
+  module: llama_stack_provider_lmeval.remote
 api_dependencies: ["inference"]
 optional_api_dependencies: []

run-inline.yaml

@@ -0,0 +1,23 @@
+version: "2"
+image_name: trustyai-lmeval
+apis:
+  - inference
+  - eval
+providers:
+  inference:
+    - provider_id: vllm
+      provider_type: remote::vllm
+      config:
+        url: ${env.VLLM_URL:=http://localhost:8080/v1}
+        max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
+        api_token: ${env.VLLM_API_TOKEN:=fake}
+        tls_verify: ${env.VLLM_TLS_VERIFY:=false}
+  eval:
+    - provider_id: trustyai_lmeval
+      provider_type: inline::trustyai_lmeval
+      config:
+         base_url: ${env.BASE_URL:=http://localhost:8321/v1}
+         use_k8s: ${env.USE_K8S:=false}
+  # server:
+  #   port: ${env.PORT:=8321}
+external_providers_dir: ./providers.d

src/llama_stack_provider_lmeval/config.py

@@ -130,11 +130,6 @@ def __post_init__(self):
         """Validate the configuration"""
         if not isinstance(self.use_k8s, bool):
             raise LMEvalConfigError("use_k8s must be a boolean")
-        if self.use_k8s is False:
-            raise LMEvalConfigError(
-                "Only Kubernetes LMEval backend is supported at the moment"
-            )
-
 
 __all__ = [
     "TLSConfig",

src/llama_stack_provider_lmeval/inline/__init__.py

@@ -0,0 +1,48 @@
+import logging
+from typing import Optional
+
+from llama_stack.apis.datatypes import Api
+from llama_stack.providers.datatypes import ProviderSpec
+
+from llama_stack_provider_lmeval.config import LMEvalBenchmarkConfig
+from .lmeval import LMEvalInline
+
+logger = logging.getLogger(__name__)
+
+async def get_provider_impl(
+    config: LMEvalBenchmarkConfig,
+    deps: Optional[dict[Api, ProviderSpec]] = None,
+) -> LMEvalInline:
+    """Get an inline Eval  implementation from the configuration.
+
+    Args:
+        config: LMEvalInlineBenchmarkConfig
+        deps: Optional[dict[Api, ProviderSpec]] = None
+
+    Returns:
+        Configured LMEval Inline implementation
+
+    Raises:
+        Exception: If configuration is invalid
+    """
+    try:
+        if deps is None:
+            deps = {}
+
+        # Extract base_url from config if available
+        base_url = None
+        if hasattr(config, "model_args") and config.model_args:
+            for arg in config.model_args:
+                if arg.get("name") == "base_url":
+                    base_url = arg.get("value")
+                    logger.debug(f"Using base_url from config: {base_url}")
+                    break
+
+        return LMEvalInline(config=config)
+    except Exception as e:
+        raise Exception(f"Failed to create LMEval implementation: {str(e)}") from e
+
+__all__ = [
+    "get_provider_impl",
+    "LMEvalInline",
+]