Merge pull request #2339 from ramalama-labs/metrics

Add benchmark metrics persistence

Author: ieaves

docs/ramalama-bench.1.md

@@ -51,6 +51,9 @@ process to be launched inside of the container. If an environment variable is
 specified without a value, the container engine checks the host environment
 for a value and set the variable only if it is set on the host.
 
+#### **--format**
+Set the output format of the benchmark results. Options include json and table (default: table).
+
 #### **--help**, **-h**
 show this help message and exit
 

docs/ramalama-benchmarks.1.md

@@ -0,0 +1,46 @@
+% ramalama-benchmarks 1
+
+## NAME
+ramalama\-benchmarks - view and interact with historical benchmark results
+
+## SYNOPSIS
+**ramalama benchmarks** [*options*] *command* [*args*...]
+
+## DESCRIPTION
+View and interact with historical benchmark results.
+Results are stored as newline-delimited JSON (JSONL) in a `benchmarks.jsonl` file.
+The storage folder is shown in `ramalama benchmarks --help` and can be
+overridden via `ramalama.benchmarks.storage_folder` in `ramalama.conf`.
+
+## OPTIONS
+
+#### **--help**, **-h**
+show this help message and exit
+
+## COMMANDS
+
+#### **list**
+list benchmark results
+
+## LIST OPTIONS
+
+#### **--limit**=LIMIT
+limit number of results to display
+
+#### **--offset**=OFFSET
+offset for pagination (default: 0)
+
+#### **--format**={table,json}
+output format (table or json) (default: table)
+
+## EXAMPLES
+
+```
+ramalama benchmarks list
+```
+
+## SEE ALSO
+**[ramalama(1)](ramalama.1.md)**, **[ramalama-bench(1)](ramalama-bench.1.md)**, **[ramalama.conf(5)](ramalama.conf.5.md)**
+
+## HISTORY
+Jan 2026, Originally compiled by Ian Eaves <ian@ramalama.com>

docs/ramalama.1.md

@@ -137,6 +137,7 @@ The default can be overridden in the ramalama.conf file.
 | Command                                           | Description                                                |
 | ------------------------------------------------- | ---------------------------------------------------------- |
 | [ramalama-bench(1)](ramalama-bench.1.md)          |benchmark specified AI Model|
+| [ramalama-benchmarks(1)](ramalama-benchmarks.1.md)|view and interact with historical benchmark results|
 | [ramalama-chat(1)](ramalama-chat.1.md)            |OpenAI chat with the specified REST API URL|
 | [ramalama-containers(1)](ramalama-containers.1.md)|list all RamaLama containers|
 | [ramalama-convert(1)](ramalama-convert.1.md)      |convert AI Models from local storage to OCI Image|

docs/ramalama.conf

@@ -221,10 +221,22 @@
 
 
 [ramalama.user]
+#
 # Suppress the interactive prompt when running on macOS with a Podman VM
 # that doesn't support GPU acceleration (e.g., applehv provider).
 # When set to true, RamaLama will automatically proceed without GPU support
 # instead of asking for confirmation.
 # Can also be set via the `RAMALAMA_USER__NO_MISSING_GPU_PROMPT` environment variable.
 #
+
+[ramalama.benchmarks]
+#storage_folder = <default store>/benchmarks
+#
+# Manually specify where to save benchmark results.
+# By default, results are stored under the default model store directory
+# in benchmarks/benchmarks.jsonl.
+# Changing `ramalama.store` does not update this; set storage_folder explicitly.
+
+
+[ramalama.user]
 #no_missing_gpu_prompt = false

docs/ramalama.conf.5.md

@@ -267,6 +267,16 @@ Configuration settings for the openai hosted provider
 **api_key**=""
 
 Provider-specific API key used when invoking OpenAI-hosted transports. Overrides `RAMALAMA_API_KEY` when set.
+## RAMALAMA.BENCHMARKS TABLE
+The ramalama.benchmarks table contains benchmark related settings.
+
+`[[ramalama.benchmarks]]`
+
+**storage_folder**="<default store>/benchmarks"
+
+Manually specify where to save benchmark results.
+By default, this will be stored in the default model store directory under `benchmarks/`.
+Changing `ramalama.store` does not update this; set `ramalama.benchmarks.storage_folder` explicitly if needed.
 
 ## RAMALAMA.USER TABLE
 The ramalama.user table contains user preference settings.

inference-spec/engines/llama.cpp.yaml

@@ -117,7 +117,22 @@ commands:
     inference_engine:
       name: "llama-bench"
       binary: "llama-bench"
-      options: *bench_perplexity_options
+      options:
+        - name: "--model"
+          description: "The AI model to run"
+          value: "{{ model.model_path }}"
+        - name: "-ngl"
+          description: "Number of layers to offload to the GPU if available"
+          value: "{{ 999 if args.ngl < 0 else args.ngl }}"
+        - name: "-ngld"
+          description: "Number of layers to offload to the GPU if available"
+          value: "{{ None if not args.model_draft else 999 if args.ngl < 0 else args.ngl }}"
+        - name: "--threads"
+          description: "Number of Threads to use during generation"
+          value: "{{ args.threads }}"
+        - name: "-o"
+          description: "Output format printed to stdout"
+          value: "json"
   - name: rag
     inference_engine:
       name: "rag"

ramalama/benchmarks/errors.py

@@ -0,0 +1,12 @@
+class MissingStorageFolderError(Exception):
+    def __init__(self):
+        message = """
+No valid benchmarks storage folder could be determined.
+
+Set an explicit path via:
+RAMALAMA__BENCHMARKS_STORAGE_FOLDER=/absolute/path/to/benchmarks
+
+If this seems wrong for your setup, report it at:
+https://www.github.com/containers/ramalama/issues
+        """
+        super().__init__(message)

ramalama/benchmarks/manager.py

@@ -0,0 +1,50 @@
+import json
+import logging
+from dataclasses import asdict
+from functools import cached_property
+from pathlib import Path
+
+from ramalama.benchmarks.errors import MissingStorageFolderError
+from ramalama.benchmarks.schemas import BenchmarkRecord, DeviceInfoV1, get_benchmark_record
+from ramalama.benchmarks.utilities import parse_jsonl
+from ramalama.config import CONFIG
+from ramalama.log_levels import LogLevel
+
+logger = logging.getLogger("ramalama.benchmarks")
+logger.setLevel(CONFIG.log_level or LogLevel.WARNING)
+
+SCHEMA_VERSION = 1
+BENCHMARKS_FILENAME = "benchmarks.jsonl"
+
+
+class BenchmarksManager:
+    def __init__(self, storage_folder: str | Path | None):
+        if storage_folder is None:
+            raise MissingStorageFolderError
+
+        self.storage_folder = Path(storage_folder)
+        self.storage_file = self.storage_folder / BENCHMARKS_FILENAME
+        self.storage_file.parent.mkdir(parents=True, exist_ok=True)
+
+    @cached_property
+    def device_info(self) -> DeviceInfoV1:
+        return DeviceInfoV1.current_device_info()
+
+    def save(self, results: list[BenchmarkRecord] | BenchmarkRecord):
+        if not isinstance(results, list):
+            results = [results]
+
+        if len(results) == 0:
+            return
+
+        with self.storage_file.open("a", encoding="utf-8") as handle:
+            for record in results:
+                handle.write(json.dumps(asdict(record), ensure_ascii=True))
+                handle.write("\n")
+
+    def list(self) -> list[BenchmarkRecord]:
+        """List benchmark results from JSONL storage."""
+        if not self.storage_file.exists():
+            return []
+        content = self.storage_file.read_text(encoding="utf-8")
+        return [get_benchmark_record(result) for result in parse_jsonl(content)]