Skip to content

Commit 43e098c

Browse files
szedan-rhszedan-rh
committed
Add E2E tests for keyword routing
This commit implements comprehensive end-to-end tests for keyword routing functionality, addressing Issue #667. Test Coverage: - OR operator (6 tests): match when any keyword is present - AND operator (5 tests): match when all keywords must be present - NOR operator (2 tests): match when no keywords are present - Case sensitivity (3 tests): case-sensitive and case-insensitive matching - Word boundaries (3 tests): prevent partial word matches - Regex special characters (3 tests): handle dots, asterisks literally - Edge cases (8 tests): empty text, Unicode, emoji, newlines - Multiple rule matching (2 tests): priority order validation - Confidence scores (1 test): keyword matches return 1.0 - JSON test data (1 test): load and validate test cases - Error handling (2 tests): invalid operators, empty keywords Test Results: - Total: 35 tests - Passing: 35 - Failing: 0 - Average test time: < 0.3ms per test - Race detection: PASS Files Added: - e2e-tests/testcases/suite_test.go: Ginkgo test suite entry point - e2e-tests/testcases/keyword_routing_test.go: 35 comprehensive tests - e2e-tests/testcases/helpers.go: Test helper functions - e2e-tests/testcases/testdata/keyword_routing_cases.json: 26 JSON test cases - e2e-tests/testcases/go.mod & go.sum: Module configuration - .github/workflows/unit-test-e2e-testcases.yml: CI/CD workflow Fixes #667 Signed-off-by: szedan Signed-off-by: Senan Zedan <[email protected]>
1 parent 33306f4 commit 43e098c

File tree

7 files changed

+1376
-0
lines changed

7 files changed

+1376
-0
lines changed

‎.github/workflows/unit-test-e2e-testcases.yml‎

Lines changed: 317 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,317 @@
1+
name: E2E Testcases Unit Tests
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- "e2e-tests/testcases/**"
7+
- "src/semantic-router/pkg/classification/**"
8+
- "src/semantic-router/pkg/config/**"
9+
- "candle-binding/**"
10+
- ".github/workflows/unit-test-e2e-testcases.yml"
11+
push:
12+
branches:
13+
- main
14+
workflow_dispatch:
15+
16+
env:
17+
GO_VERSION: '1.24'
18+
RUST_VERSION: '1.90.0'
19+
20+
jobs:
21+
test-keyword-routing:
22+
name: Keyword Routing Tests
23+
runs-on: ubuntu-latest
24+
25+
steps:
26+
- name: Checkout code
27+
uses: actions/checkout@v4
28+
29+
- name: Set up Go
30+
uses: actions/setup-go@v5
31+
with:
32+
go-version: ${{ env.GO_VERSION }}
33+
cache: true
34+
cache-dependency-path: e2e-tests/testcases/go.sum
35+
36+
- name: Set up Rust
37+
uses: actions-rust-lang/setup-rust-toolchain@v1
38+
with:
39+
toolchain: ${{ env.RUST_VERSION }}
40+
41+
- name: Cache Rust dependencies
42+
uses: actions/cache@v4
43+
with:
44+
path: |
45+
~/.cargo/bin/
46+
~/.cargo/registry/index/
47+
~/.cargo/registry/cache/
48+
~/.cargo/git/db/
49+
candle-binding/target/
50+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
51+
restore-keys: |
52+
${{ runner.os }}-cargo-
53+
54+
- name: Build Rust Candle Bindings
55+
run: |
56+
cd candle-binding
57+
cargo build --release
58+
ls -la target/release/
59+
60+
- name: Verify Rust library
61+
run: |
62+
if [ -f "candle-binding/target/release/libcandle_semantic_router.so" ]; then
63+
echo "✅ Rust library built successfully"
64+
ls -lh candle-binding/target/release/libcandle_semantic_router.so
65+
else
66+
echo "❌ Rust library not found"
67+
exit 1
68+
fi
69+
70+
- name: Run Keyword Routing Tests
71+
env:
72+
LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
73+
run: |
74+
cd e2e-tests/testcases
75+
echo "Running keyword routing tests..."
76+
go test -v -run "Keyword Routing" -coverprofile=coverage-keyword.out -covermode=atomic
77+
78+
- name: Generate coverage report
79+
if: always()
80+
run: |
81+
cd e2e-tests/testcases
82+
go tool cover -func=coverage-keyword.out > coverage-summary.txt
83+
echo "=== Coverage Summary ==="
84+
cat coverage-summary.txt
85+
86+
# Extract total coverage
87+
TOTAL_COVERAGE=$(grep "total:" coverage-summary.txt | awk '{print $3}')
88+
echo "Total coverage: $TOTAL_COVERAGE"
89+
echo "COVERAGE=$TOTAL_COVERAGE" >> $GITHUB_ENV
90+
91+
- name: Check coverage threshold
92+
if: always()
93+
run: |
94+
cd e2e-tests/testcases
95+
COVERAGE_PERCENT=$(echo $COVERAGE | sed 's/%//')
96+
THRESHOLD=80
97+
98+
if (( $(echo "$COVERAGE_PERCENT < $THRESHOLD" | bc -l) )); then
99+
echo "❌ Coverage $COVERAGE is below threshold ${THRESHOLD}%"
100+
exit 1
101+
else
102+
echo "✅ Coverage $COVERAGE meets threshold ${THRESHOLD}%"
103+
fi
104+
105+
- name: Upload coverage to Codecov
106+
uses: codecov/codecov-action@v4
107+
if: always()
108+
with:
109+
files: ./e2e-tests/testcases/coverage-keyword.out
110+
flags: e2e-testcases-keyword
111+
name: keyword-routing-coverage
112+
fail_ci_if_error: false
113+
114+
- name: Test Summary
115+
if: always()
116+
run: |
117+
echo "### Keyword Routing Test Results :test_tube:" >> $GITHUB_STEP_SUMMARY
118+
echo "" >> $GITHUB_STEP_SUMMARY
119+
echo "**Coverage:** $COVERAGE" >> $GITHUB_STEP_SUMMARY
120+
echo "" >> $GITHUB_STEP_SUMMARY
121+
echo "#### Test Categories" >> $GITHUB_STEP_SUMMARY
122+
echo "- ✅ OR operator tests" >> $GITHUB_STEP_SUMMARY
123+
echo "- ✅ AND operator tests" >> $GITHUB_STEP_SUMMARY
124+
echo "- ✅ NOR operator tests" >> $GITHUB_STEP_SUMMARY
125+
echo "- ✅ Case sensitivity tests" >> $GITHUB_STEP_SUMMARY
126+
echo "- ✅ Word boundary tests" >> $GITHUB_STEP_SUMMARY
127+
echo "- ✅ Regex special character tests" >> $GITHUB_STEP_SUMMARY
128+
echo "- ✅ Edge case tests" >> $GITHUB_STEP_SUMMARY
129+
echo "- ✅ Multiple rule matching" >> $GITHUB_STEP_SUMMARY
130+
echo "- ✅ Confidence score validation" >> $GITHUB_STEP_SUMMARY
131+
echo "- ✅ JSON test data loading" >> $GITHUB_STEP_SUMMARY
132+
echo "- ✅ Error handling" >> $GITHUB_STEP_SUMMARY
133+
134+
test-embedding-routing:
135+
name: Embedding Routing Tests
136+
runs-on: ubuntu-latest
137+
# Only run if embedding tests exist (for future PRs)
138+
if: |
139+
contains(github.event.pull_request.changed_files, 'e2e-tests/testcases/embedding_routing_test.go') ||
140+
github.event_name == 'workflow_dispatch'
141+
142+
steps:
143+
- name: Checkout code
144+
uses: actions/checkout@v4
145+
146+
- name: Set up Go
147+
uses: actions/setup-go@v5
148+
with:
149+
go-version: ${{ env.GO_VERSION }}
150+
cache: true
151+
152+
- name: Set up Rust
153+
uses: actions-rust-lang/setup-rust-toolchain@v1
154+
with:
155+
toolchain: ${{ env.RUST_VERSION }}
156+
157+
- name: Build Rust Candle Bindings
158+
run: |
159+
cd candle-binding
160+
cargo build --release
161+
162+
- name: Run Embedding Routing Tests
163+
env:
164+
LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
165+
run: |
166+
cd e2e-tests/testcases
167+
if [ -f "embedding_routing_test.go" ] && ! [[ "$(basename embedding_routing_test.go)" =~ \.skip$ ]]; then
168+
echo "Running embedding routing tests..."
169+
go test -v -run "Embedding Routing" -coverprofile=coverage-embedding.out -covermode=atomic
170+
else
171+
echo "⏭️ Embedding routing tests not ready yet (skipped)"
172+
fi
173+
174+
test-hybrid-routing:
175+
name: Hybrid Routing Tests
176+
runs-on: ubuntu-latest
177+
# Only run if hybrid tests exist (for future PRs)
178+
if: |
179+
contains(github.event.pull_request.changed_files, 'e2e-tests/testcases/hybrid_routing_test.go') ||
180+
github.event_name == 'workflow_dispatch'
181+
182+
steps:
183+
- name: Checkout code
184+
uses: actions/checkout@v4
185+
186+
- name: Set up Go
187+
uses: actions/setup-go@v5
188+
with:
189+
go-version: ${{ env.GO_VERSION }}
190+
cache: true
191+
192+
- name: Set up Rust
193+
uses: actions-rust-lang/setup-rust-toolchain@v1
194+
with:
195+
toolchain: ${{ env.RUST_VERSION }}
196+
197+
- name: Build Rust Candle Bindings
198+
run: |
199+
cd candle-binding
200+
cargo build --release
201+
202+
- name: Run Hybrid Routing Tests
203+
env:
204+
LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
205+
run: |
206+
cd e2e-tests/testcases
207+
if [ -f "hybrid_routing_test.go" ] && ! [[ "$(basename hybrid_routing_test.go)" =~ \.skip$ ]]; then
208+
echo "Running hybrid routing tests..."
209+
go test -v -run "Hybrid Routing" -coverprofile=coverage-hybrid.out -covermode=atomic
210+
else
211+
echo "⏭️ Hybrid routing tests not ready yet (skipped)"
212+
fi
213+
214+
race-detection:
215+
name: Race Condition Detection
216+
runs-on: ubuntu-latest
217+
218+
steps:
219+
- name: Checkout code
220+
uses: actions/checkout@v4
221+
222+
- name: Set up Go
223+
uses: actions/setup-go@v5
224+
with:
225+
go-version: ${{ env.GO_VERSION }}
226+
cache: true
227+
228+
- name: Set up Rust
229+
uses: actions-rust-lang/setup-rust-toolchain@v1
230+
with:
231+
toolchain: ${{ env.RUST_VERSION }}
232+
233+
- name: Build Rust Candle Bindings
234+
run: |
235+
cd candle-binding
236+
cargo build --release
237+
238+
- name: Run tests with race detector
239+
env:
240+
LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
241+
run: |
242+
cd e2e-tests/testcases
243+
echo "Running tests with race detector..."
244+
go test -race -v -run "Keyword Routing" || {
245+
echo "❌ Race conditions detected!"
246+
exit 1
247+
}
248+
echo "✅ No race conditions detected"
249+
250+
lint:
251+
name: Lint Go Code
252+
runs-on: ubuntu-latest
253+
254+
steps:
255+
- name: Checkout code
256+
uses: actions/checkout@v4
257+
258+
- name: Set up Go
259+
uses: actions/setup-go@v5
260+
with:
261+
go-version: ${{ env.GO_VERSION }}
262+
cache: true
263+
264+
- name: Run golangci-lint
265+
uses: golangci/golangci-lint-action@v6
266+
with:
267+
version: latest
268+
working-directory: e2e-tests/testcases
269+
args: --timeout=5m
270+
271+
summary:
272+
name: Test Summary
273+
if: always()
274+
runs-on: ubuntu-latest
275+
needs: [test-keyword-routing, race-detection, lint]
276+
277+
steps:
278+
- name: Check test results
279+
run: |
280+
echo "=== E2E Testcases Summary ==="
281+
echo "Keyword Routing Tests: ${{ needs.test-keyword-routing.result }}"
282+
echo "Race Detection: ${{ needs.race-detection.result }}"
283+
echo "Lint: ${{ needs.lint.result }}"
284+
285+
# Count failures
286+
FAILURES=0
287+
if [[ "${{ needs.test-keyword-routing.result }}" == "failure" ]]; then
288+
echo "❌ Keyword routing tests failed"
289+
FAILURES=$((FAILURES + 1))
290+
fi
291+
if [[ "${{ needs.race-detection.result }}" == "failure" ]]; then
292+
echo "❌ Race detection failed"
293+
FAILURES=$((FAILURES + 1))
294+
fi
295+
if [[ "${{ needs.lint.result }}" == "failure" ]]; then
296+
echo "❌ Lint failed"
297+
FAILURES=$((FAILURES + 1))
298+
fi
299+
300+
echo ""
301+
echo "=== Test Coverage (Issue #667) ==="
302+
echo "✅ OR operator - any keyword matches"
303+
echo "✅ AND operator - all keywords must match"
304+
echo "✅ NOR operator - no keywords match"
305+
echo "✅ Case-sensitive vs case-insensitive matching"
306+
echo "✅ Regex pattern matching"
307+
echo "✅ Word boundary detection"
308+
echo "✅ Priority over embedding and intent-based routing"
309+
310+
if [ $FAILURES -gt 0 ]; then
311+
echo ""
312+
echo "❌ $FAILURES test(s) failed. Check the logs for details."
313+
exit 1
314+
else
315+
echo ""
316+
echo "✅ All E2E testcases passed!"
317+
fi

‎e2e-tests/testcases/go.mod‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
module github.com/vllm-project/semantic-router/e2e-tests/testcases
2+
3+
go 1.24.1
4+
5+
require (
6+
github.com/onsi/ginkgo/v2 v2.23.4
7+
github.com/onsi/gomega v1.38.0
8+
github.com/vllm-project/semantic-router/src/semantic-router v0.0.0
9+
)
10+
11+
require (
12+
github.com/bahlo/generic-list-go v0.2.0 // indirect
13+
github.com/beorn7/perks v1.0.1 // indirect
14+
github.com/buger/jsonparser v1.1.1 // indirect
15+
github.com/cespare/xxhash/v2 v2.3.0 // indirect
16+
github.com/go-logr/logr v1.4.3 // indirect
17+
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
18+
github.com/google/go-cmp v0.7.0 // indirect
19+
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
20+
github.com/google/uuid v1.6.0 // indirect
21+
github.com/invopop/jsonschema v0.13.0 // indirect
22+
github.com/mailru/easyjson v0.7.7 // indirect
23+
github.com/mark3labs/mcp-go v0.42.0-beta.1 // indirect
24+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
25+
github.com/prometheus/client_golang v1.23.0 // indirect
26+
github.com/prometheus/client_model v0.6.2 // indirect
27+
github.com/prometheus/common v0.65.0 // indirect
28+
github.com/prometheus/procfs v0.16.1 // indirect
29+
github.com/spf13/cast v1.7.1 // indirect
30+
github.com/vllm-project/semantic-router/candle-binding v0.0.0-00010101000000-000000000000 // indirect
31+
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
32+
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
33+
go.uber.org/automaxprocs v1.6.0 // indirect
34+
go.uber.org/multierr v1.11.0 // indirect
35+
go.uber.org/zap v1.27.0 // indirect
36+
golang.org/x/net v0.43.0 // indirect
37+
golang.org/x/sys v0.37.0 // indirect
38+
golang.org/x/text v0.28.0 // indirect
39+
golang.org/x/tools v0.35.0 // indirect
40+
google.golang.org/protobuf v1.36.9 // indirect
41+
gopkg.in/yaml.v2 v2.4.0 // indirect
42+
gopkg.in/yaml.v3 v3.0.1 // indirect
43+
)
44+
45+
replace (
46+
github.com/vllm-project/semantic-router/candle-binding => ../../candle-binding
47+
github.com/vllm-project/semantic-router/src/semantic-router => ../../src/semantic-router
48+
)

0 commit comments

Comments
 (0)