Skip to content

Commit 65ce54d

Browse files
BowTiedRadoneBowTiedRadone
committed
Add property-based tests for the failure persistence module
1 parent bf4105b commit 65ce54d

File tree

1 file changed

+383
-3
lines changed

1 file changed

+383
-3
lines changed

‎persistence.tests.ts‎

Lines changed: 383 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,384 @@
1-
describe("failure persistence", () => {
2-
// TODO: Add tests for the failure persistence functionality.
3-
it("should persist failures", async () => true);
1+
import { mkdirSync, rmSync, statSync } from "fs";
2+
import { tmpdir } from "os";
3+
import { join, resolve } from "path";
4+
import {
5+
getFailureFilePath,
6+
persistFailure,
7+
loadFailures,
8+
} from "./persistence";
9+
import { RunDetails } from "./heatstroke.types";
10+
import fc from "fast-check";
11+
12+
const temporaryTestBaseDir = resolve(tmpdir(), "rendezvous-test-persistence");
13+
14+
const TEST_CONTRACT_ID = "ST1PQHQKV0RJXZFY1DGX8MNSNYVE3VGZJSRTPGZGM.counter";
15+
16+
const fileNameRegex = /^[a-zA-Z0-9_-]+$/;
17+
18+
const createTemporaryCustomTestBaseDir = (dirName: string) => {
19+
const path = join(temporaryTestBaseDir, dirName);
20+
mkdirSync(path, { recursive: true });
21+
return path;
22+
};
23+
24+
// Mock RunDetails helper
25+
const createMockRunDetails = (
26+
seed: number,
27+
failed: boolean = true
28+
): RunDetails => ({
29+
failed,
30+
seed,
31+
numRuns: 100,
32+
counterexample: [],
33+
});
34+
35+
afterAll(() => {
36+
rmSync(temporaryTestBaseDir, { recursive: true, force: true });
37+
});
38+
39+
describe("Failure Persistence", () => {
40+
describe("Failure file path", () => {
41+
it("returns the correct absolute file path with default base directory", () => {
42+
// Arrange & Act
43+
const filePath = getFailureFilePath(TEST_CONTRACT_ID);
44+
45+
// Assert
46+
expect(filePath).toBe(
47+
resolve(".rendezvous-regressions", `${TEST_CONTRACT_ID}.json`)
48+
);
49+
});
50+
51+
it("returns the correct absolute file path with any given custom base directory", () => {
52+
fc.assert(
53+
fc.property(
54+
// Arrange
55+
fc.record({ customBaseDir: fc.string() }),
56+
({ customBaseDir }) => {
57+
// Act
58+
const filePath = getFailureFilePath(
59+
TEST_CONTRACT_ID,
60+
customBaseDir
61+
);
62+
63+
// Assert
64+
expect(filePath).toBe(
65+
resolve(customBaseDir, `${TEST_CONTRACT_ID}.json`)
66+
);
67+
}
68+
)
69+
);
70+
});
71+
});
72+
73+
describe("Persisting Failures", () => {
74+
it("persisting a failure correctly creates the regression file", () => {
75+
fc.assert(
76+
fc.property(
77+
fc.record({
78+
customBaseDirName: fc.stringMatching(fileNameRegex),
79+
seed: fc.integer(),
80+
type: fc.constantFrom("invariant", "test"),
81+
}),
82+
({ customBaseDirName, seed, type }) => {
83+
// Setup
84+
const runDetails = createMockRunDetails(seed);
85+
const customBaseDir =
86+
createTemporaryCustomTestBaseDir(customBaseDirName);
87+
88+
// Exercise
89+
persistFailure(runDetails, type, TEST_CONTRACT_ID, {
90+
baseDir: customBaseDir,
91+
});
92+
93+
// Verify
94+
expect(
95+
statSync(getFailureFilePath(TEST_CONTRACT_ID, customBaseDir))
96+
).toBeDefined();
97+
98+
// Teardown
99+
rmSync(customBaseDir, {
100+
recursive: true,
101+
force: true,
102+
});
103+
}
104+
)
105+
);
106+
});
107+
108+
it("persisting failures correctly groups them by test type", () => {
109+
fc.assert(
110+
fc.property(
111+
fc.record({
112+
customBaseDirName: fc.stringMatching(fileNameRegex),
113+
seed1: fc.integer(),
114+
seed2: fc.integer(),
115+
}),
116+
({ customBaseDirName, seed1, seed2 }) => {
117+
// Setup
118+
const runDetails1 = createMockRunDetails(seed1);
119+
const runDetails2 = createMockRunDetails(seed2);
120+
const customBaseDir =
121+
createTemporaryCustomTestBaseDir(customBaseDirName);
122+
123+
// Exercise
124+
persistFailure(runDetails1, "invariant", TEST_CONTRACT_ID, {
125+
baseDir: customBaseDir,
126+
});
127+
persistFailure(runDetails2, "test", TEST_CONTRACT_ID, {
128+
baseDir: customBaseDir,
129+
});
130+
131+
// Verify
132+
const invariantFailures = loadFailures(
133+
TEST_CONTRACT_ID,
134+
"invariant",
135+
{
136+
baseDir: customBaseDir,
137+
}
138+
);
139+
const testFailures = loadFailures(TEST_CONTRACT_ID, "test", {
140+
baseDir: customBaseDir,
141+
});
142+
expect(invariantFailures).toHaveLength(1);
143+
expect(invariantFailures[0].seed).toBe(seed1);
144+
expect(testFailures).toHaveLength(1);
145+
expect(testFailures[0].seed).toBe(seed2);
146+
147+
// Teardown
148+
rmSync(customBaseDir, {
149+
recursive: true,
150+
force: true,
151+
});
152+
}
153+
)
154+
);
155+
});
156+
157+
it("persisting failures does not duplicate the same seed within a type", () => {
158+
fc.assert(
159+
fc.property(
160+
fc.record({
161+
customBaseDirName: fc.stringMatching(fileNameRegex),
162+
seed: fc.integer(),
163+
type: fc.constantFrom("invariant", "test"),
164+
}),
165+
({ customBaseDirName, seed, type }) => {
166+
// Setup
167+
const runDetails = createMockRunDetails(seed);
168+
const customBaseDir =
169+
createTemporaryCustomTestBaseDir(customBaseDirName);
170+
171+
// Exercise
172+
persistFailure(runDetails, type, TEST_CONTRACT_ID, {
173+
baseDir: customBaseDir,
174+
});
175+
persistFailure(runDetails, type, TEST_CONTRACT_ID, {
176+
baseDir: customBaseDir,
177+
});
178+
179+
// Verify
180+
const failures = loadFailures(TEST_CONTRACT_ID, type, {
181+
baseDir: customBaseDir,
182+
});
183+
expect(failures).toHaveLength(1);
184+
expect(failures[0].seed).toBe(seed);
185+
186+
// Teardown
187+
rmSync(customBaseDir, {
188+
recursive: true,
189+
force: true,
190+
});
191+
}
192+
)
193+
);
194+
});
195+
196+
it("persisting failures correctly allows same seed for different types", () => {
197+
fc.assert(
198+
fc.property(
199+
fc.record({
200+
customBaseDirName: fc.stringMatching(fileNameRegex),
201+
seed: fc.integer(),
202+
}),
203+
({ customBaseDirName, seed }) => {
204+
// Setup
205+
const runDetails = createMockRunDetails(seed);
206+
const customBaseDir =
207+
createTemporaryCustomTestBaseDir(customBaseDirName);
208+
209+
// Exercise
210+
persistFailure(runDetails, "invariant", TEST_CONTRACT_ID, {
211+
baseDir: customBaseDir,
212+
});
213+
persistFailure(runDetails, "test", TEST_CONTRACT_ID, {
214+
baseDir: customBaseDir,
215+
});
216+
217+
// Verify
218+
const invariantFailures = loadFailures(
219+
TEST_CONTRACT_ID,
220+
"invariant",
221+
{
222+
baseDir: customBaseDir,
223+
}
224+
);
225+
const testFailures = loadFailures(TEST_CONTRACT_ID, "test", {
226+
baseDir: customBaseDir,
227+
});
228+
expect(invariantFailures).toHaveLength(1);
229+
expect(invariantFailures[0].seed).toBe(seed);
230+
expect(testFailures).toHaveLength(1);
231+
expect(testFailures[0].seed).toBe(seed);
232+
233+
// Teardown
234+
rmSync(customBaseDir, {
235+
recursive: true,
236+
force: true,
237+
});
238+
}
239+
)
240+
);
241+
});
242+
243+
it("persisting failures correctly saves multiple failures of same type", () => {
244+
fc.assert(
245+
fc.property(
246+
fc.record({
247+
customBaseDirName: fc.stringMatching(fileNameRegex),
248+
// Generate between 2 and 5 seeds. If there are duplicates, only
249+
// the unique seeds should be persisted.
250+
seeds: fc.array(fc.integer(), { minLength: 2, maxLength: 5 }),
251+
}),
252+
({ customBaseDirName, seeds }) => {
253+
// Setup
254+
// Extract the unique seeds from the array.
255+
const uniqueSeeds = [...new Set(seeds)];
256+
257+
const customBaseDir =
258+
createTemporaryCustomTestBaseDir(customBaseDirName);
259+
260+
// Exercise
261+
seeds.forEach((seed) => {
262+
persistFailure(
263+
createMockRunDetails(seed),
264+
"invariant",
265+
TEST_CONTRACT_ID,
266+
{ baseDir: customBaseDir }
267+
);
268+
});
269+
270+
// Verify
271+
const failures = loadFailures(TEST_CONTRACT_ID, "invariant", {
272+
baseDir: customBaseDir,
273+
});
274+
expect(failures).toHaveLength(uniqueSeeds.length);
275+
uniqueSeeds.forEach((seed) => {
276+
expect(failures.map((f) => f.seed)).toContain(seed);
277+
});
278+
279+
// Teardown
280+
rmSync(customBaseDir, { recursive: true, force: true });
281+
}
282+
)
283+
);
284+
});
285+
286+
it("always includes valid timestamp in failure record", () => {
287+
fc.assert(
288+
fc.property(
289+
fc.record({
290+
customBaseDirName: fc.stringMatching(fileNameRegex),
291+
seed: fc.integer(),
292+
}),
293+
({ customBaseDirName, seed }) => {
294+
// Setup
295+
const before = Date.now();
296+
const runDetails = createMockRunDetails(seed);
297+
const customBaseDir =
298+
createTemporaryCustomTestBaseDir(customBaseDirName);
299+
300+
// Exercise
301+
persistFailure(runDetails, "invariant", TEST_CONTRACT_ID, {
302+
baseDir: customBaseDir,
303+
});
304+
305+
const after = Date.now();
306+
307+
// Verify
308+
const failures = loadFailures(TEST_CONTRACT_ID, "invariant", {
309+
baseDir: customBaseDir,
310+
});
311+
expect(failures[0].timestamp).toBeGreaterThanOrEqual(before);
312+
expect(failures[0].timestamp).toBeLessThanOrEqual(after);
313+
314+
// Teardown
315+
rmSync(customBaseDir, { recursive: true, force: true });
316+
}
317+
)
318+
);
319+
});
320+
});
321+
322+
describe("loadFailures", () => {
323+
it("returns empty array when no failures exist", () => {
324+
fc.assert(
325+
fc.property(
326+
fc.record({
327+
customBaseDirName: fc.stringMatching(fileNameRegex),
328+
}),
329+
({ customBaseDirName }) => {
330+
// Setup
331+
const customBaseDir =
332+
createTemporaryCustomTestBaseDir(customBaseDirName);
333+
334+
// Exercise
335+
const failures = loadFailures(TEST_CONTRACT_ID, "invariant", {
336+
baseDir: customBaseDir,
337+
});
338+
339+
// Verify
340+
expect(failures).toEqual([]);
341+
342+
// Teardown
343+
rmSync(customBaseDir, { recursive: true, force: true });
344+
}
345+
)
346+
);
347+
});
348+
349+
it("returns empty array for type with no failures", () => {
350+
fc.assert(
351+
fc.property(
352+
fc.record({
353+
customBaseDirName: fc.stringMatching(fileNameRegex),
354+
seed: fc.integer(),
355+
}),
356+
({ customBaseDirName, seed }) => {
357+
// Setup
358+
const customBaseDir =
359+
createTemporaryCustomTestBaseDir(customBaseDirName);
360+
persistFailure(
361+
createMockRunDetails(seed),
362+
"test",
363+
TEST_CONTRACT_ID,
364+
{
365+
baseDir: customBaseDir,
366+
}
367+
);
368+
369+
// Exercise
370+
const failures = loadFailures(TEST_CONTRACT_ID, "invariant", {
371+
baseDir: customBaseDir,
372+
});
373+
374+
// Verify
375+
expect(failures).toEqual([]);
376+
377+
// Teardown
378+
rmSync(customBaseDir, { recursive: true, force: true });
379+
}
380+
)
381+
);
382+
});
383+
});
4384
});

0 commit comments

Comments
 (0)