Skip to content

Commit e289cc7

Browse files
himanshusinghshimanshusinghs
committed
chore: make snapshot storage independent of accuracyRunId and commitSHA
1 parent ad87d41 commit e289cc7

File tree

8 files changed

+77
-66
lines changed

8 files changed

+77
-66
lines changed

‎scripts/mark-accuracy-run-finished.ts

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎scripts/run-accuracy-tests.sh

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,12 @@ node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPattern "
2424
# accuracyRunStatus: "in-progress". When all the tests are done and jest exits
2525
# with an exit code of 0, we can safely mark accuracy run as finished otherwise
2626
# failed.
27-
if [ $? -eq 0 ]; then
28-
MDB_ACCURACY_RUN_STATUS="done" npx tsx scripts/mark-accuracy-run-finished.ts
27+
JEST_EXIT_CODE=$?
28+
if [ $JEST_EXIT_CODE -eq 0 ]; then
29+
MDB_ACCURACY_RUN_STATUS="done" npx tsx scripts/update-accuracy-run-status.ts || echo "Warning: Failed to update accuracy run status to 'done'"
2930
else
30-
MDB_ACCURACY_RUN_STATUS="failed" npx tsx scripts/mark-accuracy-run-finished.ts
31-
fi
31+
MDB_ACCURACY_RUN_STATUS="failed" npx tsx scripts/update-accuracy-run-status.ts || echo "Warning: Failed to update accuracy run status to 'failed'"
32+
fi
33+
34+
# Preserve the original Jest exit code for CI
35+
exit $JEST_EXIT_CODE

‎scripts/update-accuracy-run-status.ts

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import { getAccuracySnapshotStorage } from "../tests/accuracy/sdk/accuracy-snapshot-storage/get-snapshot-storage.js";
2+
import {
3+
AccuracyRunStatus,
4+
AccuracyRunStatuses,
5+
} from "../tests/accuracy/sdk/accuracy-snapshot-storage/snapshot-storage.js";
6+
7+
const envAccuracyRunId = process.env.MDB_ACCURACY_RUN_ID;
8+
const envAccuracyRunStatus = process.env.MDB_ACCURACY_RUN_STATUS;
9+
10+
let status: AccuracyRunStatuses | undefined;
11+
if (
12+
!envAccuracyRunId ||
13+
(envAccuracyRunStatus !== AccuracyRunStatus.Done && envAccuracyRunStatus !== AccuracyRunStatus.Failed)
14+
) {
15+
process.exit(1);
16+
}
17+
18+
console.time(`Marked accuracy run id - ${envAccuracyRunId} as ${status} in`);
19+
const storage = await getAccuracySnapshotStorage();
20+
await storage.updateAccuracyRunStatus(envAccuracyRunId, envAccuracyRunStatus);
21+
await storage.close();
22+
console.timeEnd(`Marked accuracy run id - ${envAccuracyRunId} as ${status} in`);

‎tests/accuracy/sdk/accuracy-snapshot-storage/disk-snapshot-storage.ts

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,11 @@ const snapshotsDir = path.resolve(rootDir, ".accuracy-snapshots");
1414
export const snapshotFilePath = path.resolve(snapshotsDir, "snapshots.json");
1515

1616
export class DiskSnapshotStorage implements AccuracySnapshotStorage {
17-
private constructor(
18-
private readonly accuracyRunId: string,
19-
private readonly commitSHA: string
20-
) {}
21-
2217
async createSnapshotEntry(
2318
snapshotEntry: Pick<
2419
AccuracySnapshotEntry,
20+
| "accuracyRunId"
21+
| "commitSHA"
2522
| "provider"
2623
| "requestedModel"
2724
| "test"
@@ -38,16 +35,14 @@ export class DiskSnapshotStorage implements AccuracySnapshotStorage {
3835
): Promise<void> {
3936
const snapshotWithMeta: AccuracySnapshotEntry = {
4037
...snapshotEntry,
41-
commitSHA: this.commitSHA,
42-
accuracyRunId: this.accuracyRunId,
4338
accuracyRunStatus: AccuracyRunStatus.InProgress,
4439
createdOn: Date.now(),
4540
};
4641

4742
await this.appendAccuracySnapshot(snapshotWithMeta);
4843
}
4944

50-
async getLatestSnapshotsForCommit(commit: string): Promise<AccuracySnapshotEntry[]> {
45+
async getLatestSnapshotForCommit(commit: string): Promise<AccuracySnapshotEntry[]> {
5146
const snapshot = await this.readSnapshot();
5247
const entries = snapshot
5348
.filter((entry) => {
@@ -58,10 +53,15 @@ export class DiskSnapshotStorage implements AccuracySnapshotStorage {
5853
return latestRunId ? snapshot.filter((entry) => entry.accuracyRunId === latestRunId) : [];
5954
}
6055

61-
async updateAccuracyRunStatus(status: AccuracyRunStatuses) {
56+
async getSnapshotForAccuracyRun(accuracyRunId: string): Promise<AccuracySnapshotEntry[]> {
57+
const snapshot = await this.readSnapshot();
58+
return snapshot.filter((entry) => entry.accuracyRunId === accuracyRunId);
59+
}
60+
61+
async updateAccuracyRunStatus(accuracyRunId: string, status: AccuracyRunStatuses) {
6262
const snapshot = await this.readSnapshot();
6363
const updatedSnapshot = snapshot.map((entry) => {
64-
if (entry.accuracyRunId === this.accuracyRunId) {
64+
if (entry.accuracyRunId === accuracyRunId) {
6565
return {
6666
...entry,
6767
accuracyRunStatus: status,
@@ -116,8 +116,8 @@ export class DiskSnapshotStorage implements AccuracySnapshotStorage {
116116
return new Promise((resolve) => setTimeout(resolve, ms));
117117
}
118118

119-
static async getStorage(commitSHA: string, accuracyRunId: string) {
119+
static async getStorage() {
120120
await fs.mkdir(snapshotsDir, { recursive: true });
121-
return new DiskSnapshotStorage(commitSHA, accuracyRunId);
121+
return new DiskSnapshotStorage();
122122
}
123123
}

‎tests/accuracy/sdk/accuracy-snapshot-storage/get-snapshot-storage.ts

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,5 @@ export async function getAccuracySnapshotStorage(): Promise<AccuracySnapshotStor
1616
throw new Error("Cannot create AccuracySnapshotStorage without a commitSHA.");
1717
}
1818

19-
return (
20-
MongoDBSnapshotStorage.getStorage(commitSHA, accuracyRunId) ??
21-
(await DiskSnapshotStorage.getStorage(commitSHA, accuracyRunId))
22-
);
19+
return MongoDBSnapshotStorage.getStorage() ?? (await DiskSnapshotStorage.getStorage());
2320
}

‎tests/accuracy/sdk/accuracy-snapshot-storage/mdb-snapshot-storage.ts

Lines changed: 12 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -10,30 +10,24 @@ import {
1010
export class MongoDBSnapshotStorage implements AccuracySnapshotStorage {
1111
private readonly client: MongoClient;
1212
private readonly snapshotCollection: Collection;
13-
private readonly accuracyRunId: string;
14-
private readonly commitSHA: string;
1513
private constructor({
1614
mongodbUrl,
1715
database,
1816
collection,
19-
accuracyRunId,
20-
commitSHA,
2117
}: {
2218
mongodbUrl: string;
2319
database: string;
2420
collection: string;
25-
accuracyRunId: string;
26-
commitSHA: string;
2721
}) {
2822
this.client = new MongoClient(mongodbUrl);
2923
this.snapshotCollection = this.client.db(database).collection(collection);
30-
this.accuracyRunId = accuracyRunId;
31-
this.commitSHA = commitSHA;
3224
}
3325

3426
async createSnapshotEntry(
3527
snapshotEntry: Pick<
3628
AccuracySnapshotEntry,
29+
| "accuracyRunId"
30+
| "commitSHA"
3731
| "provider"
3832
| "requestedModel"
3933
| "test"
@@ -50,17 +44,20 @@ export class MongoDBSnapshotStorage implements AccuracySnapshotStorage {
5044
): Promise<void> {
5145
const snapshotWithMeta: AccuracySnapshotEntry = {
5246
...snapshotEntry,
53-
commitSHA: this.commitSHA,
54-
accuracyRunId: this.accuracyRunId,
5547
accuracyRunStatus: AccuracyRunStatus.InProgress,
5648
createdOn: Date.now(),
5749
};
5850
await this.snapshotCollection.insertOne(snapshotWithMeta);
5951
}
6052

61-
async getLatestSnapshotsForCommit(commit: string): Promise<AccuracySnapshotEntry[]> {
53+
async getLatestSnapshotForCommit(commit: string): Promise<AccuracySnapshotEntry[]> {
6254
const latestRunId = await this.getLatestAccuracyRunForCommit(commit);
63-
return latestRunId ? this.getSnapshotEntriesForRunId(latestRunId) : [];
55+
return latestRunId ? this.getSnapshotForAccuracyRun(latestRunId) : [];
56+
}
57+
58+
async getSnapshotForAccuracyRun(accuracyRunId: string): Promise<AccuracySnapshotEntry[]> {
59+
const snapshotEntries = await this.snapshotCollection.find({ accuracyRunId }).toArray();
60+
return AccuracySnapshotEntrySchema.array().parse(snapshotEntries);
6461
}
6562

6663
private async getLatestAccuracyRunForCommit(commit: string): Promise<string | undefined> {
@@ -72,14 +69,9 @@ export class MongoDBSnapshotStorage implements AccuracySnapshotStorage {
7269
return document?.accuracyRunId ? `${document?.accuracyRunId}` : undefined;
7370
}
7471

75-
private async getSnapshotEntriesForRunId(accuracyRunId: string): Promise<AccuracySnapshotEntry[]> {
76-
const snapshotEntries = await this.snapshotCollection.find({ accuracyRunId }).toArray();
77-
return AccuracySnapshotEntrySchema.array().parse(snapshotEntries);
78-
}
79-
80-
async updateAccuracyRunStatus(status: AccuracyRunStatuses) {
72+
async updateAccuracyRunStatus(accuracyRunId: string, status: AccuracyRunStatuses) {
8173
await this.snapshotCollection.updateMany(
82-
{ accuracyRunId: this.accuracyRunId },
74+
{ accuracyRunId: accuracyRunId },
8375
{ $set: { accuracyRunStatus: status } }
8476
);
8577
}
@@ -88,7 +80,7 @@ export class MongoDBSnapshotStorage implements AccuracySnapshotStorage {
8880
await this.client.close();
8981
}
9082

91-
static getStorage(commitSHA: string, accuracyRunId: string): MongoDBSnapshotStorage | null {
83+
static getStorage(): MongoDBSnapshotStorage | null {
9284
const mongodbUrl = process.env.MDB_ACCURACY_MDB_URL;
9385
const database = process.env.MDB_ACCURACY_MDB_DB;
9486
const collection = process.env.MDB_ACCURACY_MDB_COLLECTION;
@@ -100,8 +92,6 @@ export class MongoDBSnapshotStorage implements AccuracySnapshotStorage {
10092
mongodbUrl,
10193
database,
10294
collection,
103-
commitSHA,
104-
accuracyRunId,
10595
});
10696
}
10797
}

‎tests/accuracy/sdk/accuracy-snapshot-storage/snapshot-storage.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ export interface AccuracySnapshotStorage {
5353
createSnapshotEntry(
5454
snapshotEntry: Pick<
5555
AccuracySnapshotEntry,
56+
| "accuracyRunId"
57+
| "commitSHA"
5658
| "provider"
5759
| "requestedModel"
5860
| "test"
@@ -68,9 +70,11 @@ export interface AccuracySnapshotStorage {
6870
>
6971
): Promise<void>;
7072

71-
getLatestSnapshotsForCommit(commit: string): Promise<AccuracySnapshotEntry[]>;
73+
getLatestSnapshotForCommit(commit: string): Promise<AccuracySnapshotEntry[]>;
7274

73-
updateAccuracyRunStatus(status: AccuracyRunStatuses): Promise<void>;
75+
getSnapshotForAccuracyRun(accuracyRunId: string): Promise<AccuracySnapshotEntry[]>;
76+
77+
updateAccuracyRunStatus(accuracyRunId: string, status: AccuracyRunStatuses): Promise<void>;
7478

7579
close(): Promise<void>;
7680
}

‎tests/accuracy/sdk/describe-accuracy-tests.ts

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { prepareTestData, setupMongoDBIntegrationTest } from "../../integration/
55
import { AccuracyTestingClient, MockedTools } from "./accuracy-testing-client.js";
66
import { getAccuracySnapshotStorage } from "./accuracy-snapshot-storage/get-snapshot-storage.js";
77
import { AccuracySnapshotStorage, ExpectedToolCall } from "./accuracy-snapshot-storage/snapshot-storage.js";
8+
import { getCommitSHA } from "./git-info.js";
89

910
export interface AccuracyTestConfig {
1011
systemPrompt?: string;
@@ -26,8 +27,12 @@ export function describeAccuracyTests(
2627
[suiteName: string]: AccuracyTestConfig[];
2728
}
2829
) {
30+
if (!process.env.MDB_ACCURACY_RUN_ID) {
31+
throw new Error("MDB_ACCURACY_RUN_ID env variable is required for accuracy test runs!");
32+
}
33+
2934
if (!models.length) {
30-
throw new Error("No models available to test!");
35+
throw new Error("No models available to test. Ensure that the API keys are properly setup!");
3136
}
3237

3338
const eachModel = describe.each(models);
@@ -37,11 +42,19 @@ export function describeAccuracyTests(
3742
const mdbIntegration = setupMongoDBIntegrationTest();
3843
const { populateTestData, cleanupTestDatabases } = prepareTestData(mdbIntegration);
3944

45+
const accuracyRunId: string = `${process.env.MDB_ACCURACY_RUN_ID}`;
46+
let commitSHA: string;
4047
let accuracySnapshotStorage: AccuracySnapshotStorage;
4148
let testMCPClient: AccuracyTestingClient;
4249
let agent: VercelAgent;
4350

4451
beforeAll(async () => {
52+
const retrievedCommitSHA = await getCommitSHA();
53+
if (!retrievedCommitSHA) {
54+
throw new Error("Could not derive commitSHA, exiting accuracy tests!");
55+
}
56+
57+
commitSHA = retrievedCommitSHA;
4558
accuracySnapshotStorage = await getAccuracySnapshotStorage();
4659
testMCPClient = await AccuracyTestingClient.initializeClient(mdbIntegration.connectionString());
4760
agent = getVercelToolCallingAgent();
@@ -76,6 +89,8 @@ export function describeAccuracyTests(
7689

7790
const responseTime = timeAfterPrompt - timeBeforePrompt;
7891
await accuracySnapshotStorage.createSnapshotEntry({
92+
accuracyRunId,
93+
commitSHA,
7994
provider: model.provider,
8095
requestedModel: model.modelName,
8196
test: suiteName,

0 commit comments

Comments
 (0)