From 5960da41b73703e3e3209c9dfef592a4b6dd1cc3 Mon Sep 17 00:00:00 2001 From: Anna Cottrill Date: Mon, 9 Feb 2026 13:01:30 -0500 Subject: [PATCH 1/4] feat: record Javascript error types --- packages/dom-evaluator/src/dom-test-evaluator.ts | 1 + packages/javascript-evaluator/src/javascript-test-evaluator.ts | 1 + packages/shared/src/interfaces/test-evaluator.ts | 1 + 3 files changed, 3 insertions(+) diff --git a/packages/dom-evaluator/src/dom-test-evaluator.ts b/packages/dom-evaluator/src/dom-test-evaluator.ts index bc616b25..94b7e31c 100644 --- a/packages/dom-evaluator/src/dom-test-evaluator.ts +++ b/packages/dom-evaluator/src/dom-test-evaluator.ts @@ -73,6 +73,7 @@ export class DOMTestEvaluator implements TestEvaluator { stack: error.stack, ...(!!error.expected && { expected: error.expected }), ...(!!error.actual && { actual: error.actual }), + type: error.name, }, }; } diff --git a/packages/javascript-evaluator/src/javascript-test-evaluator.ts b/packages/javascript-evaluator/src/javascript-test-evaluator.ts index f639a04f..d3fd8af5 100644 --- a/packages/javascript-evaluator/src/javascript-test-evaluator.ts +++ b/packages/javascript-evaluator/src/javascript-test-evaluator.ts @@ -45,6 +45,7 @@ export class JavascriptTestEvaluator implements TestEvaluator { stack: error.stack, ...(!!error.expected && { expected: error.expected }), ...(!!error.actual && { actual: error.actual }), + type: error.name, }, }; } diff --git a/packages/shared/src/interfaces/test-evaluator.ts b/packages/shared/src/interfaces/test-evaluator.ts index 3894f26c..5bc42d2b 100644 --- a/packages/shared/src/interfaces/test-evaluator.ts +++ b/packages/shared/src/interfaces/test-evaluator.ts @@ -15,6 +15,7 @@ export interface TestError { expected?: unknown; actual?: unknown; type?: string; + name?: string; } export interface Fail extends Logged { From 15d38242623ef4cc9d0444c8e88b0870136f79ba Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 10 Feb 2026 08:18:13 -0500 Subject: [PATCH 2/4] Apply suggestions from code review Co-authored-by: Oliver Eyton-Williams --- packages/dom-evaluator/src/dom-test-evaluator.ts | 2 +- packages/javascript-evaluator/src/javascript-test-evaluator.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/dom-evaluator/src/dom-test-evaluator.ts b/packages/dom-evaluator/src/dom-test-evaluator.ts index 94b7e31c..a55397ec 100644 --- a/packages/dom-evaluator/src/dom-test-evaluator.ts +++ b/packages/dom-evaluator/src/dom-test-evaluator.ts @@ -73,7 +73,7 @@ export class DOMTestEvaluator implements TestEvaluator { stack: error.stack, ...(!!error.expected && { expected: error.expected }), ...(!!error.actual && { actual: error.actual }), - type: error.name, + name: error.name, }, }; } diff --git a/packages/javascript-evaluator/src/javascript-test-evaluator.ts b/packages/javascript-evaluator/src/javascript-test-evaluator.ts index d3fd8af5..a4308429 100644 --- a/packages/javascript-evaluator/src/javascript-test-evaluator.ts +++ b/packages/javascript-evaluator/src/javascript-test-evaluator.ts @@ -45,7 +45,7 @@ export class JavascriptTestEvaluator implements TestEvaluator { stack: error.stack, ...(!!error.expected && { expected: error.expected }), ...(!!error.actual && { actual: error.actual }), - type: error.name, + name: error.name, }, }; } From 51d36d6829c2291dbaa8beac1bef2ba8cd12c947 Mon Sep 17 00:00:00 2001 From: Anna Cottrill Date: Tue, 10 Feb 2026 10:29:11 -0500 Subject: [PATCH 3/4] broken tests --- packages/tests/dom-test-evaluator.test.ts | 2 ++ packages/tests/integration-tests/index.test.ts | 7 +++++++ packages/tests/javascript-test-evaluator.test.ts | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/packages/tests/dom-test-evaluator.test.ts b/packages/tests/dom-test-evaluator.test.ts index 7920d173..2489388e 100644 --- a/packages/tests/dom-test-evaluator.test.ts +++ b/packages/tests/dom-test-evaluator.test.ts @@ -38,6 +38,7 @@ describe("DOMTestEvaluator", () => { expect(result).toStrictEqual({ err: { message: "test error", + name: "Error", // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment stack: expect.stringMatching("Error: test error"), }, @@ -54,6 +55,7 @@ describe("DOMTestEvaluator", () => { message: "test error: expected 'actual' to equal 'expected'", // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment stack: expect.stringMatching("Error: test error"), + name: "AssertionError", expected: "expected", actual: "actual", }, diff --git a/packages/tests/integration-tests/index.test.ts b/packages/tests/integration-tests/index.test.ts index 1d288fa1..69b67a5c 100644 --- a/packages/tests/integration-tests/index.test.ts +++ b/packages/tests/integration-tests/index.test.ts @@ -143,6 +143,7 @@ describe("Test Runner", () => { expect(result).toEqual({ err: { message: "test error", + name: "Error", // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment stack: expect.stringMatching("Error: test error"), }, @@ -239,6 +240,7 @@ describe("Test Runner", () => { expect(result).toEqual({ err: { message: "test error", + name: "Error", // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment stack: expect.stringMatching("Error: test error"), }, @@ -586,6 +588,7 @@ new Promise((resolve) => { expect(result).toEqual({ err: { message: "afterEach error", + name: "Error", // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment stack: expect.stringMatching("Error: afterEach error"), }, @@ -1033,6 +1036,7 @@ const getFive = () => 5; err: { actual: "test", expected: "tes", + name: "AssertionError", message: "expected 'test' to equal 'tes'", // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment stack: expect.stringMatching( @@ -1108,6 +1112,7 @@ const getFive = () => 5; expect(results[0]).toEqual({ err: { actual: "NodeList []", + name: "AssertionError", // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment stack: expect.stringMatching( // Yes, there's a discrepancy between the error message and the @@ -1144,6 +1149,7 @@ const getFive = () => 5; message: expect.stringMatching( /The element has no supported sources.|Failed to load because no supported source was found./, ), + name: "NotSupportedError", }, }); }); @@ -1466,6 +1472,7 @@ checkBtn.click(); err: { actual: "Symbol(foo)", expected: "Symbol(bar)", + name: "AssertionError", message: "expected Symbol(foo) to equal Symbol(bar)", // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment stack: expect.stringMatching( diff --git a/packages/tests/javascript-test-evaluator.test.ts b/packages/tests/javascript-test-evaluator.test.ts index bbc3f8e4..8cdbd58e 100644 --- a/packages/tests/javascript-test-evaluator.test.ts +++ b/packages/tests/javascript-test-evaluator.test.ts @@ -40,6 +40,7 @@ const x = 2; message: "expected 2 to equal 1", expected: 1, actual: 2, + name: "AssertionError", // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment stack: expect.stringMatching("AssertionError: expected"), }, @@ -68,6 +69,7 @@ const x = 1; expect(result).toStrictEqual({ err: { message: "test error", + name: "Error", // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment stack: expect.stringMatching("Error: test error"), }, @@ -82,6 +84,7 @@ const x = 1; expect(result).toStrictEqual({ err: { message: "expected 'actual' to equal 'expected'", + name: "AssertionError", // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment stack: expect.stringMatching("AssertionError: expected"), expected: "expected", @@ -99,6 +102,7 @@ const x = 1; expect(result).toStrictEqual({ err: { message: "expected 1 to equal 2", + name: "AssertionError", // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment stack: expect.stringMatching("AssertionError: expected"), expected: 2, From 22a3a2e90a021ffa24d31e2f8fe81ebfe8e16ded Mon Sep 17 00:00:00 2001 From: Anna Cottrill Date: Tue, 10 Feb 2026 13:48:38 -0500 Subject: [PATCH 4/4] some kind of test --- .../tests/javascript-test-evaluator.test.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packages/tests/javascript-test-evaluator.test.ts b/packages/tests/javascript-test-evaluator.test.ts index 8cdbd58e..88a3f64d 100644 --- a/packages/tests/javascript-test-evaluator.test.ts +++ b/packages/tests/javascript-test-evaluator.test.ts @@ -202,5 +202,31 @@ assert.equal(x, 1)`; pass: true, }); }); + + it("error response should include the name of IndexOfBoundsError when thrown in user code", async () => { + evaluator.init({ + code: {}, + source: ` +x = 0;`, + }); + + const test = "assert.equal(x, 0)"; + const result = await evaluator.runTest(test); + + expect(result).toStrictEqual({ + err: { + message: "x is not defined", + name: "ReferenceError", + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + stack: expect.stringMatching("ReferenceError: x"), + }, + logs: [ + { + level: "error", + msg: "ReferenceError: x is not defined", + }, + ], + }); + }); }); });