Upload results to LLS via the Files API

Author: christinaexyou

providers.d/inline/eval/trustyai_lmeval.yaml

@@ -1,5 +1,5 @@
 module: llama_stack_provider_lmeval.inline
 config_class: llama_stack_provider_lmeval.config.LMEvalEvalProviderConfig
 pip_packages: ["lm-eval"]
-api_dependencies: ["inference"]
+api_dependencies: ["inference", "files"]
 optional_api_dependencies: []

run-inline.yaml

@@ -3,6 +3,7 @@ image_name: trustyai-lmeval
 apis:
   - inference
   - eval
+  - files
 providers:
   inference:
     - provider_id: vllm
@@ -18,4 +19,12 @@ providers:
       config:
          base_url: ${env.BASE_URL:=http://localhost:8321/v1}
          use_k8s: ${env.USE_K8S:=false}
+  files:
+    - provider_id: meta-reference-files
+      provider_type: inline::localfs
+      config:
+        storage_dir: ${env.FILES_STORAGE_DIR:=~/.llama/distributions/trustyai-lmeval/files}
+        metadata_store:
+          type: sqlite
+          db_path: ${env.METADATA_STORE_DB_PATH:=~/.llama/distributions/trustyai-lmeval}/registry.db}
 external_providers_dir: ./providers.d

src/llama_stack_provider_lmeval/config.py

@@ -12,6 +12,7 @@
 
 from .errors import LMEvalConfigError
 
+
 @json_schema_type
 @dataclass
 class TLSConfig:
@@ -128,7 +129,6 @@ class LMEvalEvalProviderConfig:
     base_dir: Path = Path(__file__).parent
     results_dir: Path = base_dir / "results"
 
-
     def __post_init__(self):
         """Validate the configuration"""
         if not isinstance(self.use_k8s, bool):

src/llama_stack_provider_lmeval/inline/__init__.py

@@ -1,3 +1,5 @@
+"""LMEval Inline Eval Llama Stack provider."""
+
 import logging
 
 from llama_stack.apis.datatypes import Api
@@ -9,6 +11,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 async def get_provider_impl(
     config: LMEvalEvalProviderConfig,
     deps: dict[Api, ProviderSpec] | None = None,
@@ -42,6 +45,7 @@ async def get_provider_impl(
     except Exception as e:
         raise RuntimeError(f"Failed to create LMEval implementation: {str(e)}") from e
 
+
 __all__ = [
     "get_provider_impl",
     "LMEvalInline",