Skip to content

Commit f1bd61b

Browse files
himanshusinghshimanshusinghs
committed
chore: use lockfile for updating local test results
1 parent 8293c67 commit f1bd61b

File tree

3 files changed

+158
-79
lines changed

3 files changed

+158
-79
lines changed

‎package-lock.json

Lines changed: 68 additions & 27 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,15 @@
3636
"license": "Apache-2.0",
3737
"devDependencies": {
3838
"@ai-sdk/azure": "^1.3.24",
39+
"@ai-sdk/google": "^1.2.22",
3940
"@ai-sdk/openai": "^1.3.23",
4041
"@eslint/js": "^9.30.1",
41-
"@ai-sdk/google": "^1.2.22",
4242
"@jest/globals": "^30.0.4",
4343
"@modelcontextprotocol/inspector": "^0.16.0",
4444
"@redocly/cli": "^1.34.4",
4545
"@types/jest": "^30.0.0",
4646
"@types/node": "^24.0.12",
47+
"@types/proper-lockfile": "^4.1.4",
4748
"@types/simple-oauth2": "^5.0.7",
4849
"@types/yargs-parser": "^21.0.3",
4950
"ai": "^4.3.17",
@@ -61,6 +62,7 @@
6162
"openapi-types": "^12.1.3",
6263
"openapi-typescript": "^7.8.0",
6364
"prettier": "^3.6.2",
65+
"proper-lockfile": "^4.1.2",
6466
"simple-git": "^3.28.0",
6567
"ts-jest": "^29.4.0",
6668
"tsx": "^4.20.3",

‎tests/accuracy/sdk/accuracy-result-storage/disk-storage.ts

Lines changed: 87 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import path from "path";
22
import fs from "fs/promises";
3+
import { lock } from "proper-lockfile";
34
import { ACCURACY_RESULTS_DIR, LATEST_ACCURACY_RUN_NAME } from "../constants.js";
45
import {
56
AccuracyResult,
@@ -25,27 +26,43 @@ export class DiskBasedResultStorage implements AccuracyResultStorage {
2526
const raw = await fs.readFile(filePath, "utf8");
2627
return JSON.parse(raw) as AccuracyResult;
2728
} catch (error) {
28-
if ((error as { code: string }).code === "ENOENT") {
29+
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
2930
return null;
3031
}
3132
throw error;
3233
}
3334
}
3435

3536
async updateRunStatus(commitSHA: string, runId: string, status: AccuracyRunStatuses): Promise<void> {
36-
await this.atomicWriteResult(commitSHA, runId, async () => {
37+
const resultFilePath = this.getAccuracyResultFilePath(commitSHA, runId);
38+
const release = await lock(resultFilePath, { retries: 10 });
39+
try {
3740
const accuracyResult = await this.getAccuracyResult(commitSHA, runId);
3841
if (!accuracyResult) {
39-
throw new Error(
40-
`Cannot update run status to ${status} for commit - ${commitSHA}, runId - ${runId}. Results not found!`
41-
);
42+
throw new Error("Results not found!");
4243
}
4344

44-
return {
45-
...accuracyResult,
46-
runStatus: status,
47-
};
48-
});
45+
await fs.writeFile(
46+
resultFilePath,
47+
JSON.stringify(
48+
{
49+
...accuracyResult,
50+
runStatus: status,
51+
},
52+
null,
53+
2
54+
),
55+
{ encoding: "utf8" }
56+
);
57+
} catch (error) {
58+
console.warn(
59+
`Could not update run status to ${status} for commit - ${commitSHA}, runId - ${runId}.`,
60+
error
61+
);
62+
throw error;
63+
} finally {
64+
await release();
65+
}
4966

5067
// This bit is important to mark the current run as the latest run for a
5168
// commit so that we can use that during baseline comparison.
@@ -63,10 +80,11 @@ export class DiskBasedResultStorage implements AccuracyResultStorage {
6380
prompt: string,
6481
modelResponse: ModelResponse
6582
): Promise<void> {
66-
await this.atomicWriteResult(commitSHA, runId, async () => {
67-
const accuracyResult = await this.getAccuracyResult(commitSHA, runId);
68-
if (!accuracyResult) {
69-
return {
83+
const resultFilePath = this.getAccuracyResultFilePath(commitSHA, runId);
84+
const { fileCreatedWithInitialData } = await this.ensureAccuracyResultFile(
85+
resultFilePath,
86+
JSON.stringify(
87+
{
7088
runId,
7189
runStatus: AccuracyRunStatus.InProgress,
7290
createdOn: Date.now(),
@@ -77,64 +95,82 @@ export class DiskBasedResultStorage implements AccuracyResultStorage {
7795
modelResponses: [modelResponse],
7896
},
7997
],
80-
};
98+
},
99+
null,
100+
2
101+
)
102+
);
103+
104+
if (fileCreatedWithInitialData) {
105+
return;
106+
}
107+
108+
const releaseLock = await lock(resultFilePath, { retries: 10 });
109+
try {
110+
const accuracyResult = await this.getAccuracyResult(commitSHA, runId);
111+
if (!accuracyResult) {
112+
throw new Error("Expected at-least initial accuracy result to be present");
81113
}
82114

83115
const existingPromptIdx = accuracyResult.promptResults.findIndex((result) => result.prompt === prompt);
84116
const promptResult = accuracyResult.promptResults[existingPromptIdx];
85117
if (!promptResult) {
86-
return {
87-
...accuracyResult,
88-
promptResults: [
89-
...accuracyResult.promptResults,
118+
return await fs.writeFile(
119+
resultFilePath,
120+
JSON.stringify(
90121
{
91-
prompt,
92-
modelResponses: [modelResponse],
122+
...accuracyResult,
123+
promptResults: [
124+
...accuracyResult.promptResults,
125+
{
126+
prompt,
127+
modelResponses: [modelResponse],
128+
},
129+
],
93130
},
94-
],
95-
};
131+
null,
132+
2
133+
)
134+
);
96135
}
97136

98137
accuracyResult.promptResults.splice(existingPromptIdx, 1, {
99138
prompt: promptResult.prompt,
100139
modelResponses: [...promptResult.modelResponses, modelResponse],
101140
});
102141

103-
return accuracyResult;
104-
});
142+
return await fs.writeFile(resultFilePath, JSON.stringify(accuracyResult, null, 2));
143+
} catch (error) {
144+
console.warn(`Could not save model response for commit - ${commitSHA}, runId - ${runId}.`, error);
145+
throw error;
146+
} finally {
147+
await releaseLock?.();
148+
}
105149
}
106150

107151
close(): Promise<void> {
108152
return Promise.resolve();
109153
}
110154

111-
private async atomicWriteResult(
112-
commitSHA: string,
113-
runId: string,
114-
generateResult: () => Promise<AccuracyResult>
115-
): Promise<void> {
116-
for (let attempt = 0; attempt < 10; attempt++) {
117-
// This should happen outside the try catch to let the result
118-
// generation error bubble up.
119-
const result = await generateResult();
120-
const resultFilePath = this.getAccuracyResultFilePath(commitSHA, runId);
121-
try {
122-
const tmp = `${resultFilePath}~${Date.now()}`;
123-
await fs.writeFile(tmp, JSON.stringify(result, null, 2));
124-
await fs.rename(tmp, resultFilePath);
125-
return;
126-
} catch (error) {
127-
if ((error as { code: string }).code === "ENOENT") {
128-
const baseDir = path.dirname(resultFilePath);
129-
await fs.mkdir(baseDir, { recursive: true });
130-
}
131-
132-
if (attempt < 10) {
133-
await this.waitFor(100 + Math.random() * 200);
134-
} else {
135-
throw error;
136-
}
155+
private async ensureAccuracyResultFile(
156+
filePath: string,
157+
initialData: string
158+
): Promise<{
159+
fileCreatedWithInitialData: boolean;
160+
}> {
161+
try {
162+
await fs.mkdir(path.dirname(filePath), { recursive: true });
163+
await fs.writeFile(filePath, initialData, { flag: "wx" });
164+
return {
165+
fileCreatedWithInitialData: true,
166+
};
167+
} catch (error) {
168+
if ((error as NodeJS.ErrnoException).code === "EEXIST") {
169+
return {
170+
fileCreatedWithInitialData: false,
171+
};
137172
}
173+
throw error;
138174
}
139175
}
140176

0 commit comments

Comments
 (0)