Skip to content

Commit aadefea

Browse files
tazarovtazarov
committed
test: add comprehensive v2 API tests and CI workflows
- Add ServerClientTest with comprehensive test coverage for v2 API - Add GitHub Actions workflow for v2 API testing with matrix strategy - Add nightly comprehensive testing workflow - Add PR validation workflow for quick checks - Add release validation workflow - Test multiple ChromaDB versions (0.5.15, 0.5.20, latest) - Test multiple Java versions (8, 11, 17, 21) - Include stress tests and performance benchmarks - Add test reporting and artifact collection
1 parent bb79d20 commit aadefea

File tree

5 files changed

+1747
-0
lines changed

5 files changed

+1747
-0
lines changed

‎.github/workflows/v2-api-nightly.yml‎

Lines changed: 368 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,368 @@
1+
name: V2 API Nightly Tests
2+
3+
on:
4+
schedule:
5+
# Run at 2 AM UTC every day
6+
- cron: '0 2 * * *'
7+
workflow_dispatch:
8+
inputs:
9+
test_experimental:
10+
description: 'Test experimental ChromaDB features'
11+
required: false
12+
type: boolean
13+
default: false
14+
15+
env:
16+
MAVEN_OPTS: -Xmx4096m -Xms1024m
17+
18+
jobs:
19+
comprehensive-v2-tests:
20+
name: Comprehensive V2 API Tests
21+
runs-on: ubuntu-latest
22+
strategy:
23+
fail-fast: false
24+
matrix:
25+
chroma-version:
26+
- '0.4.24' # Minimum supported version
27+
- '0.5.0'
28+
- '0.5.5'
29+
- '0.5.10'
30+
- '0.5.15'
31+
- '0.5.20'
32+
- 'latest'
33+
java-version: [8, 11, 17, 21]
34+
exclude:
35+
# Java 21 only with newer ChromaDB versions
36+
- java-version: 21
37+
chroma-version: '0.4.24'
38+
- java-version: 21
39+
chroma-version: '0.5.0'
40+
41+
steps:
42+
- name: Checkout code
43+
uses: actions/checkout@v4
44+
45+
- name: Set up Java ${{ matrix.java-version }}
46+
uses: actions/setup-java@v4
47+
with:
48+
java-version: ${{ matrix.java-version }}
49+
distribution: 'temurin'
50+
cache: 'maven'
51+
52+
- name: Start ChromaDB container
53+
run: |
54+
docker run -d \
55+
--name chroma-${{ matrix.chroma-version }} \
56+
-p 8000:8000 \
57+
-e ALLOW_RESET=TRUE \
58+
-e IS_PERSISTENT=FALSE \
59+
chromadb/chroma:${{ matrix.chroma-version }}
60+
61+
# Wait for ChromaDB to be ready
62+
echo "Waiting for ChromaDB to start..."
63+
for i in {1..60}; do
64+
if docker exec chroma-${{ matrix.chroma-version }} curl -f http://localhost:8000/api/v1 > /dev/null 2>&1; then
65+
echo "ChromaDB is ready!"
66+
break
67+
fi
68+
echo "Waiting... ($i/60)"
69+
sleep 2
70+
done
71+
72+
- name: Check ChromaDB health
73+
run: |
74+
curl -v http://localhost:8000/api/v1
75+
docker logs chroma-${{ matrix.chroma-version }} | tail -20
76+
77+
- name: Run V2 API tests
78+
run: |
79+
mvn clean test \
80+
-Dtest="tech.amikos.chromadb.v2.**" \
81+
-DfailIfNoTests=false \
82+
-Dchroma.url=http://localhost:8000
83+
env:
84+
CHROMA_VERSION: ${{ matrix.chroma-version }}
85+
CHROMA_URL: http://localhost:8000
86+
87+
- name: Generate detailed test report
88+
if: always()
89+
run: |
90+
mvn surefire-report:report-only
91+
mvn site -DgenerateReports=false
92+
93+
- name: Collect container logs
94+
if: failure()
95+
run: |
96+
docker logs chroma-${{ matrix.chroma-version }} > chroma-logs-${{ matrix.chroma-version }}-java-${{ matrix.java-version }}.txt
97+
98+
- name: Upload test artifacts
99+
if: always()
100+
uses: actions/upload-artifact@v3
101+
with:
102+
name: nightly-v2-chroma-${{ matrix.chroma-version }}-java-${{ matrix.java-version }}
103+
path: |
104+
target/surefire-reports/
105+
target/site/
106+
chroma-logs-*.txt
107+
108+
- name: Stop ChromaDB container
109+
if: always()
110+
run: docker stop chroma-${{ matrix.chroma-version }} && docker rm chroma-${{ matrix.chroma-version }}
111+
112+
stress-tests:
113+
name: V2 API Stress Tests
114+
runs-on: ubuntu-latest
115+
116+
services:
117+
chroma:
118+
image: chromadb/chroma:latest
119+
ports:
120+
- 8000:8000
121+
env:
122+
ALLOW_RESET: 'TRUE'
123+
IS_PERSISTENT: 'TRUE'
124+
PERSIST_DIRECTORY: '/chroma/data'
125+
options: >-
126+
--health-cmd "curl -f http://localhost:8000/api/v1 || exit 1"
127+
--health-interval 10s
128+
--health-timeout 5s
129+
--health-retries 5
130+
--mount type=tmpfs,destination=/chroma/data
131+
132+
steps:
133+
- name: Checkout code
134+
uses: actions/checkout@v4
135+
136+
- name: Set up Java 17
137+
uses: actions/setup-java@v4
138+
with:
139+
java-version: '17'
140+
distribution: 'temurin'
141+
cache: 'maven'
142+
143+
- name: Create stress test
144+
run: |
145+
cat > src/test/java/tech/amikos/chromadb/v2/V2StressTest.java << 'EOF'
146+
package tech.amikos.chromadb.v2;
147+
148+
import org.junit.BeforeClass;
149+
import org.junit.Test;
150+
import tech.amikos.chromadb.v2.auth.AuthProvider;
151+
import tech.amikos.chromadb.v2.client.Collection;
152+
import tech.amikos.chromadb.v2.client.ServerClient;
153+
import tech.amikos.chromadb.v2.model.*;
154+
155+
import java.util.*;
156+
import java.util.concurrent.*;
157+
import java.util.stream.Collectors;
158+
import java.util.stream.IntStream;
159+
160+
import static org.junit.Assert.*;
161+
162+
public class V2StressTest {
163+
private static ServerClient client;
164+
165+
@BeforeClass
166+
public static void setup() {
167+
client = ServerClient.builder()
168+
.baseUrl("http://localhost:8000")
169+
.auth(AuthProvider.none())
170+
.connectTimeout(60)
171+
.readTimeout(60)
172+
.writeTimeout(60)
173+
.build();
174+
}
175+
176+
@Test
177+
public void testLargeScale() throws Exception {
178+
String collectionName = "stress_test_" + UUID.randomUUID().toString().substring(0, 8);
179+
Collection collection = client.createCollection(collectionName);
180+
181+
// Add 10,000 records in batches of 100
182+
for (int batch = 0; batch < 100; batch++) {
183+
List<String> ids = new ArrayList<>();
184+
List<List<Float>> embeddings = new ArrayList<>();
185+
List<Map<String, Object>> metadatas = new ArrayList<>();
186+
187+
for (int i = 0; i < 100; i++) {
188+
int recordId = batch * 100 + i;
189+
ids.add("id_" + recordId);
190+
191+
// Create random embedding
192+
List<Float> embedding = new ArrayList<>();
193+
Random rand = new Random(recordId);
194+
for (int j = 0; j < 384; j++) {
195+
embedding.add(rand.nextFloat());
196+
}
197+
embeddings.add(embedding);
198+
199+
metadatas.add(Map.of(
200+
"batch", batch,
201+
"index", i,
202+
"category", "category_" + (recordId % 10)
203+
));
204+
}
205+
206+
collection.add(builder -> builder
207+
.ids(ids)
208+
.embeddings(embeddings)
209+
.metadatas(metadatas)
210+
);
211+
212+
if (batch % 10 == 0) {
213+
System.out.println("Added " + ((batch + 1) * 100) + " records");
214+
}
215+
}
216+
217+
assertEquals(10000, collection.count());
218+
System.out.println("Successfully added 10,000 records");
219+
220+
// Test queries
221+
Random rand = new Random();
222+
List<Float> queryEmbedding = IntStream.range(0, 384)
223+
.mapToObj(i -> rand.nextFloat())
224+
.collect(Collectors.toList());
225+
226+
QueryResponse result = collection.query(builder -> builder
227+
.queryEmbeddings(Arrays.asList(queryEmbedding))
228+
.nResults(100)
229+
.include(Include.METADATAS, Include.DISTANCES)
230+
);
231+
232+
assertEquals(1, result.getIds().size());
233+
assertEquals(100, result.getIds().get(0).size());
234+
235+
client.deleteCollection(collectionName);
236+
}
237+
238+
@Test
239+
public void testConcurrentOperations() throws Exception {
240+
String collectionName = "concurrent_test_" + UUID.randomUUID().toString().substring(0, 8);
241+
Collection collection = client.createCollection(collectionName);
242+
243+
ExecutorService executor = Executors.newFixedThreadPool(10);
244+
List<Future<Boolean>> futures = new ArrayList<>();
245+
246+
// Submit 100 concurrent operations
247+
for (int i = 0; i < 100; i++) {
248+
final int taskId = i;
249+
futures.add(executor.submit(() -> {
250+
try {
251+
String id = "concurrent_" + taskId;
252+
List<Float> embedding = IntStream.range(0, 384)
253+
.mapToObj(j -> (float) (taskId * 0.01))
254+
.collect(Collectors.toList());
255+
256+
collection.add(builder -> builder
257+
.ids(Arrays.asList(id))
258+
.embeddings(Arrays.asList(embedding))
259+
);
260+
return true;
261+
} catch (Exception e) {
262+
e.printStackTrace();
263+
return false;
264+
}
265+
}));
266+
}
267+
268+
// Wait for all operations to complete
269+
for (Future<Boolean> future : futures) {
270+
assertTrue(future.get(30, TimeUnit.SECONDS));
271+
}
272+
273+
executor.shutdown();
274+
executor.awaitTermination(1, TimeUnit.MINUTES);
275+
276+
assertEquals(100, collection.count());
277+
client.deleteCollection(collectionName);
278+
}
279+
}
280+
EOF
281+
282+
- name: Run stress tests
283+
run: |
284+
mvn test -Dtest=V2StressTest -DfailIfNoTests=false
285+
env:
286+
CHROMA_URL: http://localhost:8000
287+
288+
- name: Upload stress test results
289+
if: always()
290+
uses: actions/upload-artifact@v3
291+
with:
292+
name: stress-test-results-v2
293+
path: |
294+
target/surefire-reports/TEST-*V2StressTest.xml
295+
target/site/
296+
297+
report-summary:
298+
name: Generate Nightly Report
299+
needs: [comprehensive-v2-tests, stress-tests]
300+
runs-on: ubuntu-latest
301+
if: always()
302+
303+
steps:
304+
- name: Download all artifacts
305+
uses: actions/download-artifact@v3
306+
with:
307+
path: test-artifacts
308+
309+
- name: Generate summary report
310+
run: |
311+
echo "# V2 API Nightly Test Report" > nightly-report.md
312+
echo "Date: $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> nightly-report.md
313+
echo "" >> nightly-report.md
314+
315+
echo "## Test Coverage Matrix" >> nightly-report.md
316+
echo "| ChromaDB | Java 8 | Java 11 | Java 17 | Java 21 |" >> nightly-report.md
317+
echo "|----------|--------|---------|---------|---------|" >> nightly-report.md
318+
319+
# Process test results
320+
for dir in test-artifacts/nightly-v2-*; do
321+
if [ -d "$dir" ]; then
322+
basename "$dir" | grep -o "chroma-[^-]*" | cut -d- -f2 >> versions.txt
323+
fi
324+
done
325+
326+
sort -u versions.txt > unique-versions.txt || true
327+
328+
while IFS= read -r version; do
329+
if [ -n "$version" ]; then
330+
row="| $version |"
331+
for java in 8 11 17 21; do
332+
if [ -d "test-artifacts/nightly-v2-chroma-${version}-java-${java}" ]; then
333+
if ls test-artifacts/nightly-v2-chroma-${version}-java-${java}/TEST-*.xml 2>/dev/null | head -1 | xargs grep -q 'failures="0".*errors="0"' 2>/dev/null; then
334+
row="$row ✅ |"
335+
else
336+
row="$row ❌ |"
337+
fi
338+
else
339+
row="$row - |"
340+
fi
341+
done
342+
echo "$row" >> nightly-report.md
343+
fi
344+
done < unique-versions.txt || true
345+
346+
echo "" >> nightly-report.md
347+
echo "## Stress Test Results" >> nightly-report.md
348+
if [ -f "test-artifacts/stress-test-results-v2/TEST-*V2StressTest.xml" ]; then
349+
echo "✅ Stress tests completed successfully" >> nightly-report.md
350+
else
351+
echo "⚠️ Stress test results not found" >> nightly-report.md
352+
fi
353+
354+
cat nightly-report.md >> $GITHUB_STEP_SUMMARY
355+
356+
- name: Create issue for failures
357+
if: failure()
358+
uses: actions/github-script@v7
359+
with:
360+
script: |
361+
const date = new Date().toISOString().split('T')[0];
362+
await github.rest.issues.create({
363+
owner: context.repo.owner,
364+
repo: context.repo.repo,
365+
title: `[V2 API] Nightly test failures - ${date}`,
366+
body: `Nightly V2 API tests have failed. Please check the [workflow run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) for details.`,
367+
labels: ['bug', 'v2-api', 'nightly-test']
368+
});

0 commit comments

Comments
 (0)