Skip to content

Commit d882957

Browse files
authored
ci: improve testing reliability and consistency (#1907)
1 parent 87bfa89 commit d882957

File tree

10 files changed

+379
-325
lines changed

10 files changed

+379
-325
lines changed

.github/workflows/npm_release.yml

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
name: NPM Release
12
on:
23
push:
34
branches:
@@ -19,7 +20,7 @@ permissions:
1920
jobs:
2021
build:
2122
name: Build
22-
runs-on: macos-15-intel
23+
runs-on: macos-15
2324
outputs:
2425
npm_version: ${{ steps.npm_version_output.outputs.NPM_VERSION }}
2526
npm_tag: ${{ steps.npm_version_output.outputs.NPM_TAG }}
@@ -90,7 +91,7 @@ jobs:
9091
with:
9192
name: debug-symbols
9293
path: test-app/runtime/build/intermediates/merged_native_libs/release/mergeReleaseNativeLibs/out/lib/*
93-
94+
9495
test:
9596
name: Test
9697
runs-on: macos-15-intel
@@ -143,6 +144,12 @@ jobs:
143144
#target: google_apis
144145
arch: ${{env.ANDROID_ABI}}
145146
script: ./gradlew runtestsAndVerifyResults --stacktrace
147+
- name: Upload Test Results
148+
if: ${{ !cancelled() }} # run this step even if previous step failed
149+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
150+
with:
151+
name: android-unit-test-results
152+
path: test-app/dist/android_unit_test_results.xml
146153
publish:
147154
runs-on: ubuntu-latest
148155
environment: npm-publish

.github/workflows/pull_request.yml

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
name: Pull Request
12
on:
23
pull_request:
34

@@ -9,14 +10,13 @@ env:
910
ANDROID_ABI: x86_64
1011
NDK_ARCH: darwin
1112

12-
1313
permissions:
1414
contents: read
1515

1616
jobs:
1717
build:
1818
name: Build
19-
runs-on: macos-15-intel
19+
runs-on: macos-15
2020
outputs:
2121
npm_version: ${{ steps.npm_version_output.outputs.NPM_VERSION }}
2222
npm_tag: ${{ steps.npm_version_output.outputs.NPM_TAG }}
@@ -128,4 +128,10 @@ jobs:
128128
# this is needed on API 30+
129129
#target: google_apis
130130
arch: ${{env.ANDROID_ABI}}
131-
script: ./gradlew runtestsAndVerifyResults --stacktrace
131+
script: ./gradlew runtestsAndVerifyResults --stacktrace
132+
- name: Upload Test Results
133+
if: ${{ !cancelled() }} # run this step even if previous step failed
134+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
135+
with:
136+
name: android-unit-test-results
137+
path: test-app/dist/android_unit_test_results.xml

.github/workflows/test_report.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
name: "Test Report"
2+
on:
3+
workflow_run:
4+
workflows: ["Pull Request", "NPM Release"] # runs after Pull Request workflow
5+
types:
6+
- completed
7+
permissions:
8+
contents: read
9+
actions: read
10+
checks: write
11+
jobs:
12+
report:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: dorny/test-reporter@b082adf0eced0765477756c2a610396589b8c637 # v2.5.0
16+
with:
17+
name: Android Runtime Tests
18+
artifact: android-unit-test-results # artifact name
19+
path: test-app/dist/android_unit_test_results.xml
20+
reporter: jest-junit # Format of test results

test-app/app/src/main/assets/app/Infrastructure/Jasmine/jasmine-reporters/junit_reporter.js

Lines changed: 1 addition & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -215,50 +215,7 @@
215215
return;
216216
} catch (f) { errors.push(' NodeJS attempt: ' + f.message); }
217217
try {
218-
// Instead of writing XML files, output test summary to console
219-
// Parse the XML text to extract test summary
220-
var testMatch = text.match(/tests="(\d+)"/g);
221-
var failureMatch = text.match(/failures="(\d+)"/g);
222-
var errorMatch = text.match(/errors="(\d+)"/g);
223-
var skippedMatch = text.match(/skipped="(\d+)"/g);
224-
225-
var totalTests = 0;
226-
var totalFailures = 0;
227-
var totalErrors = 0;
228-
var totalSkipped = 0;
229-
230-
// Sum up all test suite results
231-
if (testMatch) {
232-
for (var i = 0; i < testMatch.length; i++) {
233-
var match = testMatch[i].match(/tests="(\d+)"/);
234-
if (match) totalTests += parseInt(match[1]);
235-
}
236-
}
237-
238-
if (failureMatch) {
239-
for (var i = 0; i < failureMatch.length; i++) {
240-
var match = failureMatch[i].match(/failures="(\d+)"/);
241-
if (match) totalFailures += parseInt(match[1]);
242-
}
243-
}
244-
245-
if (errorMatch) {
246-
for (var i = 0; i < errorMatch.length; i++) {
247-
var match = errorMatch[i].match(/errors="(\d+)"/);
248-
if (match) totalErrors += parseInt(match[1]);
249-
}
250-
}
251-
252-
if (skippedMatch) {
253-
for (var i = 0; i < skippedMatch.length; i++) {
254-
var match = skippedMatch[i].match(/skipped="(\d+)"/);
255-
if (match) totalSkipped += parseInt(match[1]);
256-
}
257-
}
258-
259-
// Output in a format our test checker can detect
260-
var resultPrefix = (totalFailures > 0 || totalErrors > 0) ? "FAILURE:" : "SUCCESS:";
261-
console.log(resultPrefix + " " + totalTests + " specs, " + (totalFailures + totalErrors) + " failures, " + totalSkipped + " skipped");
218+
__JUnitSaveResults(text);
262219

263220
return;
264221
} catch (f) {

test-app/app/src/main/assets/app/boot.js

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,21 @@ global.__onUncaughtError = function(error){
1414
}
1515

1616
require('./Infrastructure/timers');
17+
global.__JUnitSaveResults = function (unitTestResults) {
18+
var pathToApp = '/data/data/com.tns.testapplication';
19+
var unitTestFileName = 'android_unit_test_results.xml';
20+
try {
21+
var javaFile = new java.io.File(pathToApp, unitTestFileName);
22+
var stream = new java.io.FileOutputStream(javaFile);
23+
var actualEncoding = 'UTF-8';
24+
var writer = new java.io.OutputStreamWriter(stream, actualEncoding);
25+
writer.write(unitTestResults);
26+
writer.close();
27+
}
28+
catch (exception) {
29+
android.util.Log.d("TEST RESULTS", 'failed writing to files dir: ' + exception)
30+
}
31+
};
1732

1833
require('./Infrastructure/Jasmine/jasmine-2.0.1/boot'); //runs jasmine, attaches the junitOutputter
1934

test-app/app/src/main/assets/app/mainpage.js

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,4 @@ require('./tests/testURLSearchParamsImpl.js');
7373
require('./tests/testPerformanceNow');
7474
require('./tests/testQueueMicrotask');
7575

76-
// ES MODULE TESTS
77-
__log("=== Running ES Modules Tests ===");
7876
require("./tests/testESModules.mjs");
Lines changed: 33 additions & 166 deletions
Original file line numberDiff line numberDiff line change
@@ -1,167 +1,34 @@
1-
async function runESModuleTests() {
2-
let passed = 0;
3-
let failed = 0;
4-
const failureDetails = [];
5-
6-
const recordPass = (message, ...args) => {
7-
console.log(`✅ PASS: ${message}`, ...args);
8-
passed++;
9-
};
10-
11-
const recordFailure = (message, options = {}) => {
12-
const { error, details = [] } = options;
13-
const fullMessage = error?.message
14-
? `${message}: ${error.message}`
15-
: message;
16-
console.log(`❌ FAIL: ${fullMessage}`);
17-
details.forEach((detail) => console.log(detail));
18-
if (error?.stack) {
19-
console.log("Stack trace:", error.stack);
20-
}
21-
failed++;
22-
failureDetails.push(fullMessage);
23-
};
24-
25-
const logFinalSummary = () => {
26-
console.log("\n=== ES MODULE TEST RESULTS ===");
27-
console.log("Tests passed:", passed);
28-
console.log("Tests failed:", failed);
29-
console.log("Total tests:", passed + failed);
30-
31-
if (failed === 0) {
32-
console.log("ALL ES MODULE TESTS PASSED!");
33-
} else {
34-
console.log("SOME ES MODULE TESTS FAILED!");
35-
console.log("FAILURE DETECTED: Starting failure logging");
36-
failureDetails.forEach((detail) => {
37-
console.log(` ❌ ${detail}`);
38-
});
39-
}
40-
};
41-
42-
try {
43-
// Test 1: Load .mjs files as ES modules
44-
console.log("\n--- Test 1: Loading .mjs files as ES modules ---");
45-
try {
46-
const moduleExports = await import("~/testSimpleESModule.mjs");
47-
if (moduleExports) {
48-
recordPass("Module exports:", JSON.stringify(moduleExports));
49-
} else {
50-
recordFailure("ES Module loaded but exports are null");
51-
}
52-
53-
if (moduleExports?.moduleType === "ES Module") {
54-
recordPass("moduleType check passed");
55-
} else {
56-
recordFailure("moduleType check failed");
57-
}
58-
} catch (e) {
59-
recordFailure("Error loading ES module", { error: e });
60-
}
61-
62-
// Test 2: Test import.meta functionality
63-
console.log("\n--- Test 2: Testing import.meta functionality ---");
64-
try {
65-
const importMetaModule = await import("~/testImportMeta.mjs");
66-
if (
67-
importMetaModule &&
68-
importMetaModule.default &&
69-
typeof importMetaModule.default === "function"
70-
) {
71-
const metaResults = importMetaModule.default();
72-
console.log(
73-
"import.meta test results:",
74-
JSON.stringify(metaResults, null, 2)
75-
);
76-
77-
if (
78-
metaResults &&
79-
metaResults.hasImportMeta &&
80-
metaResults.hasUrl &&
81-
metaResults.hasDirname
82-
) {
83-
recordPass("import.meta properties present");
84-
console.log(" - import.meta.url:", metaResults.url);
85-
console.log(" - import.meta.dirname:", metaResults.dirname);
86-
} else {
87-
recordFailure("import.meta properties missing", {
88-
details: [
89-
` - hasImportMeta: ${metaResults?.hasImportMeta}`,
90-
` - hasUrl: ${metaResults?.hasUrl}`,
91-
` - hasDirname: ${metaResults?.hasDirname}`,
92-
],
93-
});
94-
}
95-
} else {
96-
recordFailure("import.meta module has no default export function");
97-
}
98-
} catch (e) {
99-
recordFailure("Error testing import.meta", { error: e });
100-
}
101-
102-
// Test 3: Test Worker enhancements
103-
console.log("\n--- Test 3: Testing Worker enhancements ---");
104-
try {
105-
const workerModule = await import("~/testWorkerFeatures.mjs");
106-
if (
107-
workerModule &&
108-
workerModule.testWorkerFeatures &&
109-
typeof workerModule.testWorkerFeatures === "function"
110-
) {
111-
const workerResults = workerModule.testWorkerFeatures();
112-
console.log(
113-
"Worker features test results:",
114-
JSON.stringify(workerResults, null, 2)
115-
);
116-
117-
if (
118-
workerResults &&
119-
workerResults.stringPathSupported &&
120-
workerResults.urlObjectSupported &&
121-
workerResults.tildePathSupported
122-
) {
123-
recordPass("Worker enhancement features present");
124-
console.log(
125-
" - String path support:",
126-
workerResults.stringPathSupported
127-
);
128-
console.log(
129-
" - URL object support:",
130-
workerResults.urlObjectSupported
131-
);
132-
console.log(
133-
" - Tilde path support:",
134-
workerResults.tildePathSupported
135-
);
136-
} else {
137-
recordFailure("Worker enhancement features missing", {
138-
details: [
139-
` - stringPathSupported: ${workerResults?.stringPathSupported}`,
140-
` - urlObjectSupported: ${workerResults?.urlObjectSupported}`,
141-
` - tildePathSupported: ${workerResults?.tildePathSupported}`,
142-
],
143-
});
144-
}
145-
} else {
146-
recordFailure(
147-
"Worker features module has no testWorkerFeatures function"
148-
);
149-
}
150-
} catch (e) {
151-
recordFailure("Error testing Worker features", { error: e });
152-
}
153-
} catch (unexpectedError) {
154-
recordFailure("Unexpected ES module test harness failure", {
155-
error: unexpectedError,
156-
});
157-
} finally {
158-
logFinalSummary();
159-
}
160-
161-
return { passed, failed };
162-
}
163-
164-
// Run the tests immediately (avoid top-level await for broader runtime support)
165-
runESModuleTests().catch((e) => {
166-
console.error("ES Module top-level failure:", e?.message ?? e);
1+
describe("ES Modules", () => {
2+
it("loads .mjs files as ES modules", async () => {
3+
const moduleExports = await import("~/testSimpleESModule.mjs");
4+
expect(moduleExports).toBeTruthy();
5+
expect(moduleExports?.moduleType).toBe("ES Module");
6+
});
7+
8+
it("supports import.meta functionality", async () => {
9+
const importMetaModule = await import("~/testImportMeta.mjs");
10+
expect(importMetaModule).toBeTruthy();
11+
expect(typeof importMetaModule.default).toBe("function");
12+
13+
const metaResults = importMetaModule.default();
14+
expect(metaResults).toBeTruthy();
15+
expect(metaResults.hasImportMeta).toBe(true);
16+
expect(metaResults.hasUrl).toBe(true);
17+
expect(metaResults.hasDirname).toBe(true);
18+
expect(metaResults.url).toBeTruthy();
19+
expect(metaResults.dirname).toBeTruthy();
20+
});
21+
22+
it("supports Worker enhancements", async () => {
23+
// TODO: make these tests actually be normal tests instead of just importing and checking existence
24+
const workerModule = await import("~/testWorkerFeatures.mjs");
25+
expect(workerModule).toBeTruthy();
26+
expect(typeof workerModule.testWorkerFeatures).toBe("function");
27+
28+
const workerResults = workerModule.testWorkerFeatures();
29+
expect(workerResults).toBeTruthy();
30+
expect(workerResults.stringPathSupported).toBe(true);
31+
expect(workerResults.urlObjectSupported).toBe(true);
32+
expect(workerResults.tildePathSupported).toBe(true);
33+
});
16734
});

0 commit comments

Comments
 (0)