Skip to content

Commit fd964dd

Browse files
author
Yehudit Kerido
committed
run embedding tests
Signed-off-by: Yehudit Kerido <[email protected]>
1 parent 86b6ae2 commit fd964dd

File tree

7 files changed

+1961
-0
lines changed

7 files changed

+1961
-0
lines changed

‎.github/workflows/integration-tests-embedding.yml‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
name: Integration Tests
2+
3+
# Run integration tests on:
4+
# - Nightly schedule (comprehensive validation)
5+
# - Manual trigger (for debugging/pre-merge validation)
6+
# - Push to main (after PR merge)
7+
# - PRs with 'test:integration' label (opt-in for critical changes)
8+
on:
9+
schedule:
10+
# Run nightly at 3:00 AM UTC (after main test-and-build at 2:00 AM)
11+
- cron: "0 3 * * *"
12+
13+
workflow_dispatch:
14+
inputs:
15+
test_suite:
16+
description: 'Integration test suite to run'
17+
required: false
18+
default: 'all'
19+
type: choice
20+
options:
21+
- all # All integration tests
22+
- embedding # Only embedding tests
23+
- cache # Only cache integration tests (future)
24+
25+
push:
26+
branches:
27+
- main
28+
# Only run if relevant files changed
29+
paths:
30+
- 'candle-binding/**'
31+
- 'src/semantic-router/test/integration/**'
32+
- 'src/semantic-router/pkg/classification/**'
33+
- '.github/workflows/integration-tests.yml'
34+
35+
pull_request:
36+
types: [labeled, opened, synchronize, reopened]
37+
# Only run if PR has the 'test:integration' label
38+
# Allows developers to opt-in for critical PRs
39+
40+
jobs:
41+
# Check if integration tests should run
42+
should-run:
43+
runs-on: ubuntu-latest
44+
outputs:
45+
run_tests: ${{ steps.check.outputs.run_tests }}
46+
steps:
47+
- name: Check if tests should run
48+
id: check
49+
run: |
50+
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
51+
if [[ "${{ contains(github.event.pull_request.labels.*.name, 'test:integration') }}" == "true" ]]; then
52+
echo "run_tests=true" >> $GITHUB_OUTPUT
53+
echo "PR has 'test:integration' label - running integration tests"
54+
else
55+
echo "run_tests=false" >> $GITHUB_OUTPUT
56+
echo "PR does not have 'test:integration' label - skipping integration tests"
57+
fi
58+
else
59+
echo "run_tests=true" >> $GITHUB_OUTPUT
60+
echo "Event: ${{ github.event_name }} - running integration tests"
61+
fi
62+
63+
integration-tests:
64+
needs: should-run
65+
if: needs.should-run.outputs.run_tests == 'true'
66+
runs-on: ubuntu-latest
67+
68+
steps:
69+
- name: Check out the repo
70+
uses: actions/checkout@v4
71+
72+
- name: Set up Rust
73+
uses: dtolnay/rust-toolchain@stable
74+
with:
75+
toolchain: 1.90
76+
77+
- name: Set up Go
78+
uses: actions/setup-go@v5
79+
with:
80+
go-version: "1.24"
81+
82+
- name: Install system dependencies
83+
run: |
84+
sudo apt-get update
85+
sudo apt-get install -y \
86+
make \
87+
build-essential \
88+
pkg-config
89+
90+
- name: Cache Rust dependencies
91+
uses: actions/cache@v4
92+
with:
93+
path: |
94+
~/.cargo/bin/
95+
~/.cargo/registry/index/
96+
~/.cargo/registry/cache/
97+
~/.cargo/git/db/
98+
candle-binding/target/
99+
key: ${{ runner.os }}-cargo-integration-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
100+
restore-keys: |
101+
${{ runner.os }}-cargo-integration-
102+
${{ runner.os }}-cargo-
103+
104+
- name: Cache Go dependencies
105+
uses: actions/cache@v4
106+
with:
107+
path: |
108+
~/go/pkg/mod
109+
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
110+
restore-keys: |
111+
${{ runner.os }}-go-
112+
113+
- name: Cache Integration Test Models
114+
uses: actions/cache@v4
115+
with:
116+
path: |
117+
models/Qwen3-Embedding-0.6B/
118+
models/embeddinggemma-300m/
119+
models/lora_*_model/
120+
key: ${{ runner.os }}-integration-models-${{ hashFiles('tools/make/models.mk') }}
121+
restore-keys: |
122+
${{ runner.os }}-integration-models-
123+
continue-on-error: true
124+
125+
- name: Build Rust library (CPU-only, no CUDA)
126+
run: make rust-ci
127+
128+
- name: Build Go router
129+
run: make build-router
130+
131+
- name: Install HuggingFace CLI
132+
run: |
133+
pip install -U "huggingface_hub[cli]" hf_transfer
134+
135+
- name: Download embedding models
136+
env:
137+
HF_HUB_ENABLE_HF_TRANSFER: 1
138+
HF_HUB_DISABLE_TELEMETRY: 1
139+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
140+
run: |
141+
echo "Downloading models for integration tests..."
142+
make download-models-lora
143+
144+
- name: Run embedding integration tests
145+
run: make test-embedding
146+
env:
147+
CI: true
148+
CGO_ENABLED: 1
149+
LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
150+
timeout-minutes: 30
151+
152+
- name: Run embedding benchmarks (nightly only)
153+
if: github.event_name == 'schedule'
154+
run: make bench-embedding
155+
env:
156+
CI: true
157+
CGO_ENABLED: 1
158+
LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
159+
timeout-minutes: 45
160+
161+
- name: Upload test results on failure
162+
if: failure()
163+
uses: actions/upload-artifact@v4
164+
with:
165+
name: integration-test-results
166+
path: |
167+
**/*.log
168+
**/test-output.*
169+
retention-days: 7
170+
171+
- name: Comment PR with results (if PR)
172+
if: github.event_name == 'pull_request' && always()
173+
uses: actions/github-script@v7
174+
with:
175+
script: |
176+
const status = '${{ job.status }}';
177+
const emoji = status === 'success' ? '✅' : '❌';
178+
const message = `${emoji} Integration Tests: **${status.toUpperCase()}**
179+
180+
- Embedding concurrency tests: ${status === 'success' ? 'PASSED' : 'FAILED'}
181+
- Embedding memory tests: ${status === 'success' ? 'PASSED' : 'FAILED'}
182+
- Embedding performance tests: ${status === 'success' ? 'PASSED' : 'FAILED'}
183+
184+
View details: [Workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})`;
185+
186+
github.rest.issues.createComment({
187+
issue_number: context.issue.number,
188+
owner: context.repo.owner,
189+
repo: context.repo.repo,
190+
body: message
191+
});
192+
193+
- name: Notify on failure
194+
if: failure() && github.event_name == 'schedule'
195+
run: |
196+
echo "::error::Integration tests failed on nightly run. Please investigate."

0 commit comments

Comments
 (0)