Fix failing tests

Author: christinaexyou

src/llama_stack_provider_lmeval/inline/lmeval.py

@@ -10,11 +10,11 @@
 from pathlib import Path
 from typing import Any
 
-from llama_stack.apis.datatypes import Api
-from llama_stack.apis.files import OpenAIFileObject, OpenAIFilePurpose, UploadFile
 from llama_stack.apis.benchmarks import Benchmark, ListBenchmarksResponse
 from llama_stack.apis.common.job_types import Job, JobStatus
+from llama_stack.apis.datatypes import Api
 from llama_stack.apis.eval import BenchmarkConfig, Eval, EvaluateResponse
+from llama_stack.apis.files import OpenAIFileObject, OpenAIFilePurpose, UploadFile
 from llama_stack.apis.scoring import ScoringResult
 from llama_stack.providers.datatypes import BenchmarksProtocolPrivate
 
@@ -34,10 +34,12 @@ def __init__(
         self.benchmarks: dict[str, Benchmark] = {}
         self._jobs: list[Job] = []
         self._job_metadata: dict[str, dict[str, str]] = {}
-        self.files_api = deps.get(Api.files)
+        self.files_api = deps.get(Api.files) if deps else None
 
     async def initialize(self):
         "Initialize the LMEval Inline provider"
+        if not self.files_api:
+            raise LMEvalConfigError("Files API is not initialized")
 
     async def list_benchmarks(self) -> ListBenchmarksResponse:
         """List all registered benchmarks."""
@@ -59,7 +61,6 @@ def _get_job_id(self) -> str:
     async def run_eval(
         self, benchmark_id: str, benchmark_config: BenchmarkConfig, limit="2"
     ) -> Job:
-
         if not isinstance(benchmark_config, BenchmarkConfig):
             raise LMEvalConfigError("LMEval requires BenchmarkConfig")
 
@@ -109,7 +110,7 @@ async def run_eval(
                 env=env,
             )
 
-            self._job_metadata[job_id]["process_id"] = process.pid
+            self._job_metadata[job_id]["process_id"] = str(process.pid)
 
             stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=300)
 
@@ -150,9 +151,9 @@ async def run_eval(
                         )
 
                         if upload_job_result:
-                            self._job_metadata[job_id][
-                                "uploaded_file"
-                            ] = upload_job_result.id
+                            self._job_metadata[job_id]["uploaded_file"] = (
+                                upload_job_result.id
+                            )
                             logger.info(
                                 "Uploaded job result file %s to Files API with ID: %s",
                                 actual_result_file,
@@ -170,9 +171,9 @@ async def run_eval(
                             "Failed to process results file for job %s: %s", job_id, e
                         )
                         job.status = JobStatus.failed
-                        self._job_metadata[job_id][
-                            "error"
-                        ] = f"Failed to process results: {str(e)}"
+                        self._job_metadata[job_id]["error"] = (
+                            f"Failed to process results: {str(e)}"
+                        )
                 else:
                     logger.warning(
                         "No results files found for job %s in directory %s",
@@ -189,9 +190,7 @@ async def run_eval(
                 logger.error("stdout: %s", stdout.decode("utf-8") if stdout else "")
                 logger.error("stderr: %s", stderr.decode("utf-8") if stderr else "")
                 job.status = JobStatus.failed
-                self._job_metadata[job_id][
-                    "error"
-                ] = f"""
+                self._job_metadata[job_id]["error"] = f"""
                     Process failed with return code {process.returncode}
                 """
         except Exception as e:
@@ -209,6 +208,10 @@ async def run_eval(
     async def _upload_file(
         self, file: Path, purpose: OpenAIFilePurpose
     ) -> OpenAIFileObject | None:
+        if self.files_api is None:
+            logger.warning("Files API not available, cannot upload file %s", file)
+            return None
+
         if file.exists():
             with open(file, "rb") as f:
                 upload_file = await self.files_api.openai_upload_file(
@@ -239,7 +242,7 @@ async def _parse_job_results_from_data(
                     if isinstance(task_results, dict):
                         # Extract metric scores
                         for metric_name, metric_value in task_results.items():
-                            if isinstance(metric_value, (int, float)):
+                            if isinstance(metric_value, int | float):
                                 score_key = f"{task_name}:{metric_name}"
                                 scores[score_key] = ScoringResult(
                                     aggregated_results={
@@ -278,7 +281,6 @@ async def _parse_job_results_from_data(
             return EvaluateResponse(generations=[], scores={})
 
     def _create_model_args(self, base_url: str, benchmark_config: BenchmarkConfig):
-
         model_args = {"model": None, "base_url": base_url}
 
         model_name = None
@@ -333,6 +335,29 @@ def _collect_lmeval_args(
 
         return lmeval_args
 
+    def _extract_task_name(self, benchmark_id: str) -> str:
+        """Extract task name from benchmark ID.
+
+        Args:
+            benchmark_id: The benchmark identifier
+
+        Returns:
+            Task name
+
+        Raises:
+            LMEvalTaskNameError: If task name is empty or invalid
+        """
+        from ..errors import LMEvalTaskNameError
+
+        task_name_parts = benchmark_id.split("::")
+        task_name = task_name_parts[-1].strip() if task_name_parts else ""
+        if not task_name:
+            raise LMEvalTaskNameError(
+                f"Invalid benchmark_id '{benchmark_id}': task name is empty or invalid"
+            )
+
+        return task_name
+
     def build_command(
         self,
         task_config: BenchmarkConfig,
@@ -410,10 +435,7 @@ def build_command(
                 cmd.extend(["--model_args", ",".join(model_args_list)])
 
         # Extract task name from benchmark_id (remove provider prefix)
-        # benchmark_id format: "inline::trustyai_lmeval::task_name"
-        task_name = (
-            benchmark_id.split("::")[-1] if "::" in benchmark_id else benchmark_id
-        )
+        task_name = self._extract_task_name(benchmark_id)
         cmd.extend(["--tasks", task_name])
 
         cmd.extend(["--limit", limit])
@@ -475,9 +497,9 @@ async def job_cancel(self, benchmark_id: str, job_id: str) -> None:
             return
 
         if job.status in [JobStatus.in_progress, JobStatus.scheduled]:
-            process_id = self._job_metadata.get(job_id, {}).get("process_id")
-            if process_id:
-                process_id = int(process_id)
+            process_id_str = self._job_metadata.get(job_id, {}).get("process_id")
+            if process_id_str:
+                process_id = int(process_id_str)
                 logger.info("Attempting to cancel subprocess %s", process_id)
 
                 try:
@@ -573,7 +595,7 @@ async def shutdown(self) -> None:
         self.benchmarks.clear()
 
         # Close files API connection if it exists and has cleanup methods
-        if self.files_api and hasattr(self.files_api, 'close'):
+        if self.files_api and hasattr(self.files_api, "close"):
             try:
                 await self.files_api.close()
                 logger.debug("Closed Files API connection")

src/llama_stack_provider_lmeval/remote/provider.py

@@ -5,6 +5,7 @@
     remote_provider_spec,
 )
 
+
 def get_provider_spec() -> ProviderSpec:
     return remote_provider_spec(
         api=Api.eval,