Skip to content

Commit 9505901

Browse files
lucaromagnolilucaromagnoli
authored
Merge pull request #45 from lucaromagnoli/ci/security-workflow
Ci/security workflow
2 parents bb59ef8 + a267b15 commit 9505901

File tree

4 files changed

+306
-0
lines changed

4 files changed

+306
-0
lines changed

‎.github/workflows/security.yml‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
name: Security
2+
3+
on:
4+
push:
5+
branches: ["**"]
6+
paths-ignore:
7+
- '**.md'
8+
- 'docs/**'
9+
- 'LICENSE'
10+
schedule:
11+
- cron: '0 3 * * 1' # Weekly, Mondays at 03:00 UTC
12+
workflow_dispatch:
13+
14+
permissions:
15+
contents: read
16+
security-events: write
17+
actions: read
18+
pull-requests: read
19+
20+
jobs:
21+
secret-file-guard:
22+
name: Secret file guard
23+
runs-on: ubuntu-latest
24+
steps:
25+
- uses: actions/checkout@v4
26+
with:
27+
fetch-depth: 0
28+
- name: Fail if private key-like files present
29+
run: |
30+
echo "Scanning workspace for private key material (*.pem, *.p12, *.pfx, *.key)..."
31+
if find . -type f \( -name "*.pem" -o -name "*.p12" -o -name "*.pfx" -o -name "*.key" \) | grep -q .; then
32+
echo "❌ Private key-like files found in workspace."
33+
find . -type f \( -name "*.pem" -o -name "*.p12" -o -name "*.pfx" -o -name "*.key" \) -print
34+
exit 1
35+
fi
36+
echo "✅ No private key files detected."
37+
38+
codeql:
39+
name: CodeQL (C/C++)
40+
runs-on: ubuntu-latest
41+
steps:
42+
- name: Checkout
43+
uses: actions/checkout@v4
44+
45+
- name: Initialize CodeQL
46+
uses: github/codeql-action/init@v3
47+
with:
48+
languages: cpp
49+
50+
- name: Configure build
51+
run: |
52+
cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DLLMCPP_BUILD_TESTS=OFF
53+
54+
- name: Build
55+
run: cmake --build build --parallel
56+
57+
- name: Perform CodeQL Analysis
58+
uses: github/codeql-action/analyze@v3
59+
60+
gitleaks:
61+
name: Gitleaks (Secret scanning)
62+
continue-on-error: ${{ github.ref != 'refs/heads/main' }}
63+
runs-on: ubuntu-latest
64+
steps:
65+
- uses: actions/checkout@v4
66+
with:
67+
fetch-depth: 0
68+
- name: Install gitleaks
69+
run: |
70+
set -euo pipefail
71+
VERSION=8.24.3
72+
curl -sSL -o gitleaks.tgz "https://github.com/gitleaks/gitleaks/releases/download/v${VERSION}/gitleaks_${VERSION}_linux_x64.tar.gz"
73+
tar -xzf gitleaks.tgz gitleaks
74+
sudo mv gitleaks /usr/local/bin/gitleaks
75+
gitleaks version
76+
- name: Run gitleaks (generate SARIF)
77+
env:
78+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
79+
run: |
80+
set -euo pipefail
81+
gitleaks detect --source . --redact --report-format sarif --report-path gitleaks.sarif
82+
- name: Upload SARIF to code scanning
83+
uses: github/codeql-action/upload-sarif@v3
84+
if: always() && hashFiles('gitleaks.sarif') != ''
85+
with:
86+
sarif_file: gitleaks.sarif
87+
88+
# Dependency Review is PR-only by design; omit to avoid duplicate PR checks
89+
90+

‎tests/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ set(INTEGRATION_TEST_SOURCES
3434
add_executable(llmcpp_tests
3535
${UNIT_TEST_SOURCES}
3636
${INTEGRATION_TEST_SOURCES}
37+
bench/benchmark_core.cpp
3738
)
3839

3940
# Link against the library and test framework

‎tests/bench/benchmark_core.cpp‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#include <catch2/catch_test_macros.hpp>
2+
#include <catch2/benchmark/catch_benchmark.hpp>
3+
4+
#include "openai/OpenAITypes.h"
5+
6+
using namespace OpenAI;
7+
8+
// Simple helper to build a representative ResponsesRequest
9+
static ResponsesRequest makeRequest() {
10+
ResponsesRequest r;
11+
r.model = "gpt-4o";
12+
r.instructions = "Answer briefly.";
13+
r.maxOutputTokens = 128;
14+
r.toolChoice = ToolChoiceMode::Auto;
15+
16+
// Add a small JSON schema
17+
json schema = json::parse(R"({
18+
"type":"object",
19+
"properties":{"answer":{"type":"string"}},
20+
"required":["answer"]
21+
})");
22+
r.text = TextOutputConfig("answer_schema", schema, true);
23+
24+
// Add a couple of input messages
25+
std::vector<InputMessage> messages;
26+
InputMessage sys;
27+
sys.role = InputMessage::Role::System;
28+
sys.content = "You are helpful.";
29+
messages.push_back(sys);
30+
31+
InputMessage usr;
32+
usr.role = InputMessage::Role::User;
33+
usr.content = "Hello";
34+
messages.push_back(usr);
35+
36+
r.input = ResponsesInput::fromContentList(messages);
37+
return r;
38+
}
39+
40+
TEST_CASE("Benchmark: ResponsesRequest serialization", "[benchmark]") {
41+
auto req = makeRequest();
42+
BENCHMARK("toJson serialize") {
43+
return req.toJson();
44+
};
45+
}
46+
47+
TEST_CASE("Benchmark: ResponsesResponse parsing", "[benchmark]") {
48+
// Minimal example response JSON
49+
auto sample = json::parse(R"({
50+
"id": "resp_123",
51+
"object": "response",
52+
"created_at": 0,
53+
"status": "completed",
54+
"model": "gpt-4o-mini-2024-07-18",
55+
"usage": {"input_tokens": 10, "output_tokens": 5},
56+
"output": [
57+
{"type":"message","id":"msg_1","role":"assistant","content":[{"type":"output_text","text":"Hi"}]}
58+
]
59+
})");
60+
61+
BENCHMARK("fromJson parse") {
62+
return ResponsesResponse::fromJson(sample);
63+
};
64+
}
65+
66+
TEST_CASE("Benchmark: Model enum conversions", "[benchmark]") {
67+
BENCHMARK("modelToString") {
68+
return toString(Model::GPT_4o);
69+
};
70+
BENCHMARK("stringToModel") {
71+
return modelFromString("gpt-4o");
72+
};
73+
}
74+
75+

‎tests/integration/test_benchmarks.cpp‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
#include <catch2/catch_test_macros.hpp>
2+
#include <chrono>
3+
#include <cstdlib>
4+
#include <iostream>
5+
#include <string>
6+
7+
#include "openai/OpenAIClient.h"
8+
#include "openai/OpenAITypes.h"
9+
10+
using namespace std::chrono;
11+
12+
static bool isReasoningModel(OpenAI::Model model) {
13+
return model == OpenAI::Model::GPT_5 || model == OpenAI::Model::GPT_5_Mini ||
14+
model == OpenAI::Model::GPT_5_Nano || model == OpenAI::Model::O3 ||
15+
model == OpenAI::Model::O3_Mini || model == OpenAI::Model::O1 ||
16+
model == OpenAI::Model::O1_Mini || model == OpenAI::Model::O1_Preview ||
17+
model == OpenAI::Model::O1_Pro || model == OpenAI::Model::O4_Mini;
18+
}
19+
20+
TEST_CASE("OpenAI model benchmarks (structured outputs)", "[openai][integration][benchmark]") {
21+
const char* runBenchEnv = std::getenv("LLMCPP_RUN_BENCHMARKS");
22+
if (!runBenchEnv || std::string(runBenchEnv) != "1") {
23+
SUCCEED("Benchmarks skipped. Set LLMCPP_RUN_BENCHMARKS=1 to enable.");
24+
return;
25+
}
26+
27+
const char* apiKey = std::getenv("OPENAI_API_KEY");
28+
REQUIRE(apiKey != nullptr);
29+
30+
OpenAI::OpenAIClient client(apiKey);
31+
32+
// Minimal structured output schema
33+
json schema = {{"type", "object"},
34+
{"properties", {{"answer", {{"type", "string"}}}}},
35+
{"required", json::array({"answer"})},
36+
{"additionalProperties", false}};
37+
38+
// Simple input
39+
auto input = OpenAI::ResponsesInput::fromText("Reply with the word OK.");
40+
41+
// Iterate through response-capable models
42+
for (const auto& modelName : OpenAI::RESPONSES_MODELS) {
43+
DYNAMIC_SECTION("Benchmark model: " << modelName) {
44+
OpenAI::ResponsesRequest req;
45+
req.model = modelName;
46+
req.input = input;
47+
req.text = OpenAI::TextOutputConfig("bench_schema", schema, true);
48+
req.maxOutputTokens = 16;
49+
50+
// Tweak reasoning parameters when appropriate
51+
auto modelEnum = OpenAI::modelFromString(modelName);
52+
if (isReasoningModel(modelEnum)) {
53+
req.reasoning = json{{"effort", "low"}};
54+
}
55+
56+
const auto start = steady_clock::now();
57+
auto response = client.sendResponsesRequest(req);
58+
const auto end = steady_clock::now();
59+
60+
const auto elapsedMs = duration_cast<milliseconds>(end - start).count();
61+
std::cout << "[BENCH] model=" << modelName << ", ms=" << elapsedMs
62+
<< ", success=" << (response.isCompleted() && !response.hasError()) << std::endl;
63+
64+
// Sanity: we should at least get a response object back; don't assert success to avoid
65+
// flakes
66+
REQUIRE(!response.id.empty());
67+
}
68+
}
69+
}
70+
71+
#include <catch2/catch_test_macros.hpp>
72+
#include <chrono>
73+
#include <cstdlib>
74+
#include <iostream>
75+
#include <string>
76+
77+
#include "openai/OpenAIClient.h"
78+
#include "openai/OpenAITypes.h"
79+
80+
using namespace std::chrono;
81+
82+
static bool isReasoningModel(OpenAI::Model model) {
83+
return model == OpenAI::Model::GPT_5 || model == OpenAI::Model::GPT_5_Mini ||
84+
model == OpenAI::Model::GPT_5_Nano || model == OpenAI::Model::O3 ||
85+
model == OpenAI::Model::O3_Mini || model == OpenAI::Model::O1 ||
86+
model == OpenAI::Model::O1_Mini || model == OpenAI::Model::O1_Preview ||
87+
model == OpenAI::Model::O1_Pro || model == OpenAI::Model::O4_Mini;
88+
}
89+
90+
TEST_CASE("OpenAI model benchmarks (structured outputs)", "[openai][integration][benchmark]") {
91+
const char* runBenchEnv = std::getenv("LLMCPP_RUN_BENCHMARKS");
92+
if (!runBenchEnv || std::string(runBenchEnv) != "1") {
93+
SUCCEED("Benchmarks skipped. Set LLMCPP_RUN_BENCHMARKS=1 to enable.");
94+
return;
95+
}
96+
97+
const char* apiKey = std::getenv("OPENAI_API_KEY");
98+
REQUIRE(apiKey != nullptr);
99+
100+
OpenAIClient client(apiKey);
101+
102+
// Minimal structured output schema
103+
json schema = {{"type", "object"},
104+
{"properties", {{"answer", {{"type", "string"}}}}},
105+
{"required", json::array({"answer"})},
106+
{"additionalProperties", false}};
107+
108+
// Simple input
109+
auto input = OpenAI::ResponsesInput::fromText("Reply with the word OK.");
110+
111+
// Iterate through response-capable models
112+
for (const auto& modelName : OpenAI::RESPONSES_MODELS) {
113+
DYNAMIC_SECTION("Benchmark model: " << modelName) {
114+
OpenAI::ResponsesRequest req;
115+
req.model = modelName;
116+
req.input = input;
117+
req.text = OpenAI::TextOutputConfig("bench_schema", schema, true);
118+
req.maxOutputTokens = 16;
119+
120+
// Tweak reasoning parameters when appropriate
121+
auto modelEnum = OpenAI::modelFromString(modelName);
122+
if (isReasoningModel(modelEnum)) {
123+
req.reasoning = json{{"effort", "low"}};
124+
}
125+
126+
const auto start = steady_clock::now();
127+
auto response = client.sendResponsesRequest(req);
128+
const auto end = steady_clock::now();
129+
130+
const auto elapsedMs = duration_cast<milliseconds>(end - start).count();
131+
std::cout << "[BENCH] model=" << modelName << ", ms=" << elapsedMs
132+
<< ", success=" << (response.isCompleted() && !response.hasError())
133+
<< std::endl;
134+
135+
// Sanity: we should at least get a response object back; don't assert success to avoid
136+
// flakes
137+
REQUIRE(!response.id.empty());
138+
}
139+
}
140+
}

0 commit comments

Comments
 (0)