Initial PR for performance test on integration test that running on CI

.github/workflows/performance-nightly.yml

@@ -0,0 +1,136 @@
+name: Nightly Performance Baseline
+
+on:
+  schedule:
+    # Run at 3:00 AM UTC daily
+    - cron: "0 3 * * *"
+  workflow_dispatch:  # Allow manual triggering
+
+jobs:
+  update-baseline:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.24"
+
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: 1.90
+
+      - name: Cache Rust dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            candle-binding/target/
+          key: ${{ runner.os }}-nightly-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-nightly-cargo-
+
+      - name: Cache Go dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-nightly-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-nightly-go-
+
+      - name: Cache Models
+        uses: actions/cache@v4
+        with:
+          path: |
+            models/
+          key: ${{ runner.os }}-models-v1-${{ hashFiles('tools/make/models.mk') }}
+          restore-keys: |
+            ${{ runner.os }}-models-v1-
+
+      - name: Build Rust library (CPU-only)
+        run: make rust-ci
+
+      - name: Install HuggingFace CLI
+        run: |
+          pip install -U "huggingface_hub[cli]" hf_transfer
+
+      - name: Download models (full set for nightly)
+        env:
+          CI_MINIMAL_MODELS: false
+          HF_HUB_ENABLE_HF_TRANSFER: 1
+          HF_HUB_DISABLE_TELEMETRY: 1
+        run: make download-models
+
+      - name: Run comprehensive benchmarks
+        run: |
+          export LD_LIBRARY_PATH=${PWD}/candle-binding/target/release
+          cd perf
+          go test -bench=. -benchmem -benchtime=30s ./benchmarks/... | tee ../reports/nightly-bench.txt
+
+      - name: Update baselines
+        run: |
+          make perf-baseline-update
+
+      - name: Check for baseline changes
+        id: check_changes
+        run: |
+          git add perf/testdata/baselines/
+          if git diff --cached --quiet; then
+            echo "changes=false" >> $GITHUB_OUTPUT
+            echo "No baseline changes detected"
+          else
+            echo "changes=true" >> $GITHUB_OUTPUT
+            echo "Baseline changes detected"
+          fi
+
+      - name: Commit updated baselines
+        if: steps.check_changes.outputs.changes == 'true'
+        run: |
+          git config user.name "GitHub Actions Bot"
+          git config user.email "[email protected]"
+          git commit -m "chore: update performance baselines (nightly run)"
+          git push
+
+      - name: Upload nightly results
+        uses: actions/upload-artifact@v4
+        with:
+          name: nightly-baseline-${{ github.run_number }}
+          path: |
+            reports/
+            perf/testdata/baselines/
+          retention-days: 90
+
+      - name: Create issue on failure
+        if: failure()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const title = '🔥 Nightly Performance Baseline Update Failed';
+            const body = `
+            The nightly performance baseline update failed.
+
+            **Run:** ${{ github.run_id }}
+            **Time:** ${new Date().toISOString()}
+
+            Please investigate the failure in the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}).
+            `;
+
+            await github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title: title,
+              body: body,
+              labels: ['performance', 'ci-failure']
+            });

.github/workflows/performance-test.yml

@@ -0,0 +1,191 @@
+name: Performance Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'src/semantic-router/**'
+      - 'candle-binding/**'
+      - 'perf/**'
+      - '.github/workflows/performance-test.yml'
+  workflow_dispatch:
+
+jobs:
+  component-benchmarks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Need full history for baseline comparison
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.24"
+
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: 1.90
+
+      - name: Cache Rust dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            candle-binding/target/
+          key: ${{ runner.os }}-perf-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-perf-cargo-
+
+      - name: Cache Go dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-perf-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-perf-go-
+
+      - name: Cache Models
+        uses: actions/cache@v4
+        with:
+          path: |
+            models/
+          key: ${{ runner.os }}-models-v1-${{ hashFiles('tools/make/models.mk') }}
+          restore-keys: |
+            ${{ runner.os }}-models-v1-
+        continue-on-error: true
+
+      - name: Build Rust library (CPU-only)
+        run: make rust-ci
+
+      - name: Install HuggingFace CLI
+        run: |
+          pip install -U "huggingface_hub[cli]" hf_transfer
+
+      - name: Download models (minimal)
+        env:
+          CI_MINIMAL_MODELS: true
+          HF_HUB_ENABLE_HF_TRANSFER: 1
+          HF_HUB_DISABLE_TELEMETRY: 1
+        run: make download-models
+
+      - name: Download performance baselines
+        continue-on-error: true
+        run: |
+          mkdir -p perf/testdata/baselines
+          git show main:perf/testdata/baselines/classification.json > perf/testdata/baselines/classification.json 2>/dev/null || echo '{"version":"v1.0.0","benchmarks":{}}' > perf/testdata/baselines/classification.json
+          git show main:perf/testdata/baselines/decision.json > perf/testdata/baselines/decision.json 2>/dev/null || echo '{"version":"v1.0.0","benchmarks":{}}' > perf/testdata/baselines/decision.json
+          git show main:perf/testdata/baselines/cache.json > perf/testdata/baselines/cache.json 2>/dev/null || echo '{"version":"v1.0.0","benchmarks":{}}' > perf/testdata/baselines/cache.json
+
+      - name: Run component benchmarks
+        run: |
+          mkdir -p reports
+          export LD_LIBRARY_PATH=${PWD}/candle-binding/target/release
+          make perf-bench-quick 2>&1 | tee reports/bench-output.txt
+
+      - name: Parse benchmark results
+        id: parse
+        continue-on-error: true
+        run: |
+          # Extract benchmark results
+          # This is a simplified parser - a real implementation would be more robust
+          echo "benchmarks_completed=true" >> $GITHUB_OUTPUT
+
+      - name: Generate performance summary
+        id: summary
+        run: |
+          cat > reports/summary.md <<'EOF'
+          ## Performance Benchmark Results
+
+          Component benchmarks completed successfully.
+
+          ### Summary
+          - Classification benchmarks: ✅
+          - Decision engine benchmarks: ✅
+          - Cache benchmarks: ✅
+
+          ### Details
+          See attached benchmark artifacts for detailed results and profiles.
+
+          ---
+          _Performance testing powered by [vLLM Semantic Router](https://github.com/vllm-project/semantic-router)_
+          EOF
+
+      - name: Comment PR with results
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            let summary = '## Performance Benchmark Results\n\n';
+
+            try {
+              summary = fs.readFileSync('reports/summary.md', 'utf8');
+            } catch (err) {
+              summary += '✅ Component benchmarks completed\n\n';
+              summary += '_Detailed results available in workflow artifacts_\n';
+            }
+
+            // Find existing comment
+            const {data: comments} = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+
+            const botComment = comments.find(comment =>
+              comment.user.type === 'Bot' &&
+              comment.body.includes('Performance Benchmark Results')
+            );
+
+            if (botComment) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: summary
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: summary
+              });
+            }
+
+      - name: Upload performance artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance-results-${{ github.run_number }}
+          path: |
+            reports/
+          retention-days: 30
+
+      - name: Check for regressions (placeholder)
+        id: regression_check
+        continue-on-error: true
+        run: |
+          # In a real implementation, this would:
+          # 1. Parse benchmark output
+          # 2. Compare against baselines
+          # 3. Calculate % changes
+          # 4. Exit 1 if regressions exceed thresholds
+          echo "No regressions detected (placeholder check)"
+
+      - name: Fail on regression
+        if: steps.regression_check.outcome == 'failure'
+        run: |
+          echo "❌ Performance regressions detected!"
+          echo "See benchmark results in artifacts for details"
+          exit 1

Makefile

@@ -21,6 +21,7 @@ _run:
 		-f tools/make/observability.mk \
 		-f tools/make/openshift.mk \
 		-f tools/make/e2e.mk \
+		-f tools/make/performance.mk \
 		$(MAKECMDGOALS)
 
 .PHONY: _run