Add E2E tests for keyword routing (Issue #667)

.github/workflows/unit-test-e2e-testcases.yml

@@ -0,0 +1,322 @@
+name: E2E Testcases Unit Tests
+
+on:
+  pull_request:
+    paths:
+      - "e2e-tests/testcases/**"
+      - "src/semantic-router/pkg/classification/**"
+      - "src/semantic-router/pkg/config/**"
+      - "candle-binding/**"
+      - ".github/workflows/unit-test-e2e-testcases.yml"
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  GO_VERSION: '1.24'
+  RUST_VERSION: '1.90.0'
+
+jobs:
+  test-keyword-routing:
+    name: Keyword Routing Tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+          cache-dependency-path: e2e-tests/testcases/go.sum
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_VERSION }}
+
+      - name: Cache Rust dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            candle-binding/target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-
+
+      - name: Build Rust Candle Bindings
+        run: |
+          cd candle-binding
+          cargo build --release --no-default-features
+          ls -la target/release/
+
+      - name: Verify Rust library
+        run: |
+          if [ -f "candle-binding/target/release/libcandle_semantic_router.so" ]; then
+            echo "✅ Rust library built successfully"
+            ls -lh candle-binding/target/release/libcandle_semantic_router.so
+          else
+            echo "❌ Rust library not found"
+            exit 1
+          fi
+
+      - name: Run Keyword Routing Tests
+        env:
+          LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
+        run: |
+          cd e2e-tests/testcases
+          echo "Running keyword routing tests..."
+          go test -v -coverprofile=coverage-keyword.out -covermode=atomic -coverpkg=github.com/vllm-project/semantic-router/src/semantic-router/pkg/classification
+
+      - name: Generate coverage report
+        if: always()
+        run: |
+          cd e2e-tests/testcases
+          go tool cover -func=coverage-keyword.out > coverage-summary.txt
+          echo "=== Full Coverage Summary ==="
+          cat coverage-summary.txt
+
+          echo ""
+          echo "=== Keyword Classifier Coverage ==="
+          grep "keyword_classifier.go" coverage-summary.txt || echo "No keyword_classifier.go coverage found"
+
+          # Extract coverage for keyword_classifier.go only
+          # Filter lines containing keyword_classifier.go, extract percentage, calculate average
+          KEYWORD_COVERAGE=$(grep "keyword_classifier.go" coverage-summary.txt | awk '{gsub(/%/, "", $NF); sum+=$NF; count++} END {if(count>0) printf "%.1f", sum/count; else print "0.0"}')
+          echo "Keyword Classifier Average Coverage: ${KEYWORD_COVERAGE}%"
+          echo "COVERAGE=${KEYWORD_COVERAGE}%" >> $GITHUB_ENV
+
+      - name: Check coverage threshold
+        if: always()
+        run: |
+          cd e2e-tests/testcases
+          COVERAGE_PERCENT=$(echo $COVERAGE | sed 's/%//')
+          THRESHOLD=80
+
+          if (( $(echo "$COVERAGE_PERCENT < $THRESHOLD" | bc -l) )); then
+            echo "❌ Coverage $COVERAGE is below threshold ${THRESHOLD}%"
+            exit 1
+          else
+            echo "✅ Coverage $COVERAGE meets threshold ${THRESHOLD}%"
+          fi
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        if: always()
+        with:
+          files: ./e2e-tests/testcases/coverage-keyword.out
+          flags: e2e-testcases-keyword
+          name: keyword-routing-coverage
+          fail_ci_if_error: false
+
+      - name: Test Summary
+        if: always()
+        run: |
+          echo "### Keyword Routing Test Results :test_tube:" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Coverage:** $COVERAGE" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "#### Test Categories" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ OR operator tests" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ AND operator tests" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ NOR operator tests" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Case sensitivity tests" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Word boundary tests" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Regex special character tests" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Edge case tests" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Multiple rule matching" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Confidence score validation" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ JSON test data loading" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Error handling" >> $GITHUB_STEP_SUMMARY
+
+  test-embedding-routing:
+    name: Embedding Routing Tests
+    runs-on: ubuntu-latest
+    # Only run if embedding tests exist (for future PRs)
+    if: |
+      contains(github.event.pull_request.changed_files, 'e2e-tests/testcases/embedding_routing_test.go') ||
+      github.event_name == 'workflow_dispatch'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_VERSION }}
+
+      - name: Build Rust Candle Bindings
+        run: |
+          cd candle-binding
+          cargo build --release --no-default-features
+
+      - name: Run Embedding Routing Tests
+        env:
+          LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
+        run: |
+          cd e2e-tests/testcases
+          if [ -f "embedding_routing_test.go" ] && ! [[ "$(basename embedding_routing_test.go)" =~ \.skip$ ]]; then
+            echo "Running embedding routing tests..."
+            go test -v -run "Embedding Routing" -coverprofile=coverage-embedding.out -covermode=atomic
+          else
+            echo "⏭️  Embedding routing tests not ready yet (skipped)"
+          fi
+
+  test-hybrid-routing:
+    name: Hybrid Routing Tests
+    runs-on: ubuntu-latest
+    # Only run if hybrid tests exist (for future PRs)
+    if: |
+      contains(github.event.pull_request.changed_files, 'e2e-tests/testcases/hybrid_routing_test.go') ||
+      github.event_name == 'workflow_dispatch'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_VERSION }}
+
+      - name: Build Rust Candle Bindings
+        run: |
+          cd candle-binding
+          cargo build --release --no-default-features
+
+      - name: Run Hybrid Routing Tests
+        env:
+          LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
+        run: |
+          cd e2e-tests/testcases
+          if [ -f "hybrid_routing_test.go" ] && ! [[ "$(basename hybrid_routing_test.go)" =~ \.skip$ ]]; then
+            echo "Running hybrid routing tests..."
+            go test -v -run "Hybrid Routing" -coverprofile=coverage-hybrid.out -covermode=atomic
+          else
+            echo "⏭️  Hybrid routing tests not ready yet (skipped)"
+          fi
+
+  race-detection:
+    name: Race Condition Detection
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_VERSION }}
+
+      - name: Build Rust Candle Bindings
+        run: |
+          cd candle-binding
+          cargo build --release --no-default-features
+
+      - name: Run tests with race detector
+        env:
+          LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
+        run: |
+          cd e2e-tests/testcases
+          echo "Running tests with race detector..."
+          go test -race -v || {
+            echo "❌ Race conditions detected!"
+            exit 1
+          }
+          echo "✅ No race conditions detected"
+
+  lint:
+    name: Lint Go Code
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          version: latest
+          working-directory: e2e-tests/testcases
+          args: --timeout=5m
+
+  summary:
+    name: Test Summary
+    if: always()
+    runs-on: ubuntu-latest
+    needs: [test-keyword-routing, race-detection, lint]
+
+    steps:
+      - name: Check test results
+        run: |
+          echo "=== E2E Testcases Summary ==="
+          echo "Keyword Routing Tests: ${{ needs.test-keyword-routing.result }}"
+          echo "Race Detection: ${{ needs.race-detection.result }}"
+          echo "Lint: ${{ needs.lint.result }}"
+
+          # Count failures
+          FAILURES=0
+          if [[ "${{ needs.test-keyword-routing.result }}" == "failure" ]]; then
+            echo "❌ Keyword routing tests failed"
+            FAILURES=$((FAILURES + 1))
+          fi
+          if [[ "${{ needs.race-detection.result }}" == "failure" ]]; then
+            echo "❌ Race detection failed"
+            FAILURES=$((FAILURES + 1))
+          fi
+          if [[ "${{ needs.lint.result }}" == "failure" ]]; then
+            echo "❌ Lint failed"
+            FAILURES=$((FAILURES + 1))
+          fi
+
+          echo ""
+          echo "=== Test Coverage (Issue #667) ==="
+          echo "✅ OR operator - any keyword matches"
+          echo "✅ AND operator - all keywords must match"
+          echo "✅ NOR operator - no keywords match"
+          echo "✅ Case-sensitive vs case-insensitive matching"
+          echo "✅ Regex pattern matching"
+          echo "✅ Word boundary detection"
+          echo "✅ Priority over embedding and intent-based routing"
+
+          if [ $FAILURES -gt 0 ]; then
+            echo ""
+            echo "❌ $FAILURES test(s) failed. Check the logs for details."
+            exit 1
+          else
+            echo ""
+            echo "✅ All E2E testcases passed!"
+          fi

e2e-tests/testcases/go.mod

@@ -0,0 +1,48 @@
+module github.com/vllm-project/semantic-router/e2e-tests/testcases
+
+go 1.24.1
+
+require (
+	github.com/onsi/ginkgo/v2 v2.23.4
+	github.com/onsi/gomega v1.38.0
+	github.com/vllm-project/semantic-router/src/semantic-router v0.0.0
+)
+
+require (
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/invopop/jsonschema v0.13.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mark3labs/mcp-go v0.42.0-beta.1 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_golang v1.23.0 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.65.0 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/vllm-project/semantic-router/candle-binding v0.0.0-00010101000000-000000000000 // indirect
+	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
+	github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
+	go.uber.org/automaxprocs v1.6.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	golang.org/x/net v0.43.0 // indirect
+	golang.org/x/sys v0.37.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
+	golang.org/x/tools v0.35.0 // indirect
+	google.golang.org/protobuf v1.36.9 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+replace (
+	github.com/vllm-project/semantic-router/candle-binding => ../../candle-binding
+	github.com/vllm-project/semantic-router/src/semantic-router => ../../src/semantic-router
+)