Save LM-Eval results locally

Author: christinaexyou

run-inline.yaml

@@ -18,6 +18,4 @@ providers:
       config:
          base_url: ${env.BASE_URL:=http://localhost:8321/v1}
          use_k8s: ${env.USE_K8S:=false}
-  # server:
-  #   port: ${env.PORT:=8321}
 external_providers_dir: ./providers.d

src/llama_stack_provider_lmeval/config.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import Any
 
 from llama_stack.apis.eval import BenchmarkConfig, EvalCandidate
@@ -11,7 +12,6 @@
 
 from .errors import LMEvalConfigError
 
-
 @json_schema_type
 @dataclass
 class TLSConfig:
@@ -125,12 +125,16 @@ class LMEvalEvalProviderConfig:
     metadata: dict[str, Any] | None = None
     # TLS configuration - structured approach
     tls: TLSConfig | None = None
+    base_dir: Path = Path(__file__).parent
+    results_dir: Path = base_dir / "results"
+
 
     def __post_init__(self):
         """Validate the configuration"""
         if not isinstance(self.use_k8s, bool):
             raise LMEvalConfigError("use_k8s must be a boolean")
 
+
 __all__ = [
     "TLSConfig",
     "LMEvalBenchmarkConfig",

src/llama_stack_provider_lmeval/inline/__init__.py

@@ -1,23 +1,23 @@
 import logging
-from typing import Optional
 
 from llama_stack.apis.datatypes import Api
 from llama_stack.providers.datatypes import ProviderSpec
 
-from llama_stack_provider_lmeval.config import LMEvalBenchmarkConfig
+from llama_stack_provider_lmeval.config import LMEvalEvalProviderConfig
+
 from .lmeval import LMEvalInline
 
 logger = logging.getLogger(__name__)
 
 async def get_provider_impl(
-    config: LMEvalBenchmarkConfig,
-    deps: Optional[dict[Api, ProviderSpec]] = None,
+    config: LMEvalEvalProviderConfig,
+    deps: dict[Api, ProviderSpec] | None = None,
 ) -> LMEvalInline:
     """Get an inline Eval  implementation from the configuration.
 
     Args:
-        config: LMEvalInlineBenchmarkConfig
-        deps: Optional[dict[Api, ProviderSpec]] = None
+        config: LMEvalEvalProviderConfig
+        deps: Optional[dict[Api, Any]] = None - can be ProviderSpec or API instances
 
     Returns:
         Configured LMEval Inline implementation
@@ -35,12 +35,12 @@ async def get_provider_impl(
             for arg in config.model_args:
                 if arg.get("name") == "base_url":
                     base_url = arg.get("value")
-                    logger.debug(f"Using base_url from config: {base_url}")
+                    logger.debug("Using base_url from config: %s", base_url)
                     break
 
-        return LMEvalInline(config=config)
+        return LMEvalInline(config=config, deps=deps)
     except Exception as e:
-        raise Exception(f"Failed to create LMEval implementation: {str(e)}") from e
+        raise RuntimeError(f"Failed to create LMEval implementation: {str(e)}") from e
 
 __all__ = [
     "get_provider_impl",