Merge pull request #47 from lucaromagnoli/feat/model-benchmarks

tests(bench): add test_benchmarks.cpp to the test suite and remove ob…

Author: lucaromagnoli

include/openai/OpenAITypes.h

@@ -677,7 +677,8 @@ struct ResponsesRequest {
         auto modelEnum = modelFromString(model);
 
         // Reasoning models (O-series + GPT-5) have different parameter support
-        if (modelEnum == Model::GPT_5 || modelEnum == Model::O3 || modelEnum == Model::O3_Mini ||
+        if (modelEnum == Model::GPT_5 || modelEnum == Model::GPT_5_Mini ||
+            modelEnum == Model::GPT_5_Nano || modelEnum == Model::O3 || modelEnum == Model::O3_Mini ||
             modelEnum == Model::O1 || modelEnum == Model::O1_Mini ||
             modelEnum == Model::O1_Preview || modelEnum == Model::O1_Pro ||
             modelEnum == Model::O4_Mini || modelEnum == Model::O4_Mini_Deep_Research) {
@@ -1024,8 +1025,11 @@ std::string getRecommendedApiForModel(const std::string& model);
 
 // Model lists for different APIs
 const std::vector<std::string> RESPONSES_MODELS = {
-    "gpt-5",       "gpt-4o", "gpt-4o-mini", "gpt-4.1", "gpt-4.1-nano", "gpt-4.1-mini",
-    "gpt-image-1", "o1",     "o3-mini",     "o3",      "o4-mini",      "computer-use-preview"};
+    "gpt-5",       "gpt-5-mini",          "gpt-5-nano",
+    "gpt-4o",      "gpt-4o-mini",
+    "gpt-4.1",     "gpt-4.1-nano",       "gpt-4.1-mini",
+    "gpt-image-1", "o1",                  "o3-mini",     "o3",
+    "o4-mini",     "computer-use-preview"};
 
 const std::vector<std::string> CHAT_COMPLETION_MODELS = {"gpt-4", "gpt-4-turbo", "gpt-4o",
                                                          "gpt-4o-mini", "gpt-3.5-turbo"};

tests/CMakeLists.txt

@@ -35,6 +35,7 @@ add_executable(llmcpp_tests
     ${UNIT_TEST_SOURCES}
     ${INTEGRATION_TEST_SOURCES}
     bench/benchmark_core.cpp
+    bench/test_benchmarks.cpp
 )
 
 # Link against the library and test framework

tests/bench/run_model_benchmarks.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Resolve repo root (prefer git); fallback to two levels up from this script
+if repo_root=$(git rev-parse --show-toplevel 2>/dev/null); then
+  :
+else
+  script_dir="$(cd "$(dirname "$0")" && pwd)"
+  repo_root="$(cd "$script_dir/../.." && pwd)"
+fi
+
+cd "$repo_root"
+
+# Load API key from .env if present
+if [[ ! -f .env && -z "${OPENAI_API_KEY:-}" ]]; then
+  echo "ERROR: OPENAI_API_KEY not set and .env missing." >&2
+  exit 1
+fi
+# shellcheck disable=SC1091
+set -a && source .env 2>/dev/null || true && set +a
+
+build_dir=cmake-build-bench
+cmake -S . -B "$build_dir" -G Ninja -DCMAKE_BUILD_TYPE=Release -DLLMCPP_BUILD_TESTS=ON >/dev/null
+cmake --build "$build_dir" -j >/dev/null
+
+echo "model,ms,status,input_tokens,output_tokens,total_tokens"
+LLMCPP_RUN_BENCHMARKS=1 "$build_dir"/tests/llmcpp_tests "[openai][integration][benchmark]" --reporter compact 2>/dev/null |
+  awk -F, '/^gpt|^o[13]-|^o4-mini/ { print $0 }'
+
+

tests/bench/test_benchmarks.cpp

@@ -0,0 +1,87 @@
+#include <catch2/catch_test_macros.hpp>
+#include <chrono>
+#include <cstdlib>
+#include <iostream>
+#include <string>
+
+#include "openai/OpenAIClient.h"
+#include "openai/OpenAITypes.h"
+
+using namespace std::chrono;
+
+static bool isReasoningModel(OpenAI::Model model) {
+    return model == OpenAI::Model::GPT_5 || model == OpenAI::Model::GPT_5_Mini ||
+           model == OpenAI::Model::GPT_5_Nano || model == OpenAI::Model::O3 ||
+           model == OpenAI::Model::O3_Mini || model == OpenAI::Model::O1 ||
+           model == OpenAI::Model::O1_Mini || model == OpenAI::Model::O1_Preview ||
+           model == OpenAI::Model::O1_Pro || model == OpenAI::Model::O4_Mini;
+}
+
+static bool isExcludedModel(const std::string &modelName) {
+    return modelName == "gpt-image-1" || modelName == "computer-use-preview";
+}
+
+static bool isGpt5Family(OpenAI::Model model) {
+    return model == OpenAI::Model::GPT_5 || model == OpenAI::Model::GPT_5_Mini ||
+           model == OpenAI::Model::GPT_5_Nano;
+}
+
+TEST_CASE("OpenAI model benchmarks (structured outputs)", "[openai][integration][benchmark]") {
+    const char* runBenchEnv = std::getenv("LLMCPP_RUN_BENCHMARKS");
+    if (!runBenchEnv || std::string(runBenchEnv) != "1") {
+        SUCCEED("Benchmarks skipped. Set LLMCPP_RUN_BENCHMARKS=1 to enable.");
+        return;
+    }
+
+    const char* apiKey = std::getenv("OPENAI_API_KEY");
+    REQUIRE(apiKey != nullptr);
+
+    OpenAIClient client(apiKey);
+
+    // Minimal structured output schema
+    json schema = {{"type", "object"},
+                   {"properties", {{"answer", {{"type", "string"}}}}},
+                   {"required", json::array({"answer"})},
+                   {"additionalProperties", false}};
+
+    // Simple input aligned with structured output requirement
+    auto input = OpenAI::ResponsesInput::fromText(
+        "Return a JSON object that conforms to the provided schema with answer set to 'OK'.");
+
+    // Iterate through response-capable models
+    for (const auto& modelName : OpenAI::RESPONSES_MODELS) {
+        if (isExcludedModel(modelName)) {
+            std::cout << "[BENCH] skipping model=" << modelName << " (not supported for JSON schema bench)" << std::endl;
+            continue;
+        }
+        DYNAMIC_SECTION("Benchmark model: " << modelName) {
+            OpenAI::ResponsesRequest req;
+            req.model = modelName;
+            req.input = input;
+            req.text = OpenAI::TextOutputConfig("bench_schema", schema, true);
+            // Do not set max_output_tokens at all; let the server decide for all models
+            auto modelEnum = OpenAI::modelFromString(modelName);
+
+            // Tweak reasoning parameters when appropriate
+            if (isReasoningModel(modelEnum)) {
+                req.reasoning = json{{"effort", "low"}};
+            }
+
+            const auto start = steady_clock::now();
+            auto response = client.sendResponsesRequest(req);
+            const auto end = steady_clock::now();
+
+            const auto elapsedMs = duration_cast<milliseconds>(end - start).count();
+            const bool ok = (response.isCompleted() && !response.hasError());
+            int inTok = response.usage.inputTokens;
+            int outTok = response.usage.outputTokens;
+            int totalTok = inTok + outTok;
+            std::cout << modelName << "," << elapsedMs << "," << (ok ? "ok" : "fail")
+                      << "," << inTok << "," << outTok << "," << totalTok << std::endl;
+
+            // Sanity: we should at least get a response object back; don't assert success to avoid
+            // flakes
+            REQUIRE(!response.id.empty());
+        }
+    }
+}