diff --git a/.gitignore b/.gitignore index 35b0d83..9cf2fc3 100644 --- a/.gitignore +++ b/.gitignore @@ -107,4 +107,7 @@ dist *.mp4 # Browser snapshots -browser-snapshots \ No newline at end of file +browser-snapshots + +# Test output artifacts +test/artifacts/output/ \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 15330a8..482a737 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "dependencies": { "@apidevtools/json-schema-ref-parser": "^14.2.1", "@ffmpeg-installer/ffmpeg": "^1.1.0", - "@puppeteer/browsers": "^2.10.12", + "@puppeteer/browsers": "^2.10.13", "ajv": "^8.17.1", "appium": "^2.19.0", "appium-chromium-driver": "1.5.1", @@ -27,7 +27,7 @@ "json-schema-faker": "^0.5.9", "pixelmatch": "^5.3.0", "pngjs": "^7.0.0", - "posthog-node": "^5.10.4", + "posthog-node": "^5.11.0", "sharp": "^0.34.4", "tree-kill": "^1.2.2", "webdriverio": "8.45.0" @@ -2049,9 +2049,9 @@ } }, "node_modules/@puppeteer/browsers": { - "version": "2.10.12", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.12.tgz", - "integrity": "sha512-mP9iLFZwH+FapKJLeA7/fLqOlSUwYpMwjR1P5J23qd4e7qGJwecJccJqHYrjw33jmIZYV4dtiTHPD/J+1e7cEw==", + "version": "2.10.13", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.13.tgz", + "integrity": "sha512-a9Ruw3j3qlnB5a/zHRTkruppynxqaeE4H9WNj5eYGRWqw0ZauZ23f4W2ARf3hghF5doozyD+CRtt7XSYuYRI/Q==", "license": "Apache-2.0", "dependencies": { "debug": "^4.4.3", diff --git a/package.json b/package.json index 41a52e9..7339c60 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "dependencies": { "@apidevtools/json-schema-ref-parser": "^14.2.1", "@ffmpeg-installer/ffmpeg": "^1.1.0", - "@puppeteer/browsers": "^2.10.12", + "@puppeteer/browsers": "^2.10.13", "ajv": "^8.17.1", "appium": "^2.19.0", "appium-chromium-driver": "1.5.1", @@ -44,7 +44,7 @@ "json-schema-faker": "^0.5.9", "pixelmatch": "^5.3.0", "pngjs": "^7.0.0", - "posthog-node": "^5.10.4", + "posthog-node": "^5.11.0", "sharp": "^0.34.4", "tree-kill": "^1.2.2", "webdriverio": "8.45.0" diff --git a/src/tests.js b/src/tests.js index bb6c442..bb1125e 100644 --- a/src/tests.js +++ b/src/tests.js @@ -505,7 +505,7 @@ async function runSpecs({ resolvedTests }) { context.platform }", "apps": ${JSON.stringify(context.apps)}}` ); - contextReport = { result: { status: "SKIPPED" }, ...contextReport }; + contextReport = { result: "SKIPPED", ...contextReport }; report.summary.contexts.skipped++; testReport.contexts.push(contextReport); continue; @@ -563,7 +563,8 @@ async function runSpecs({ resolvedTests }) { " Make sure you've run `safaridriver --enable` in a terminal and enabled 'Allow Remote Automation' in Safari's Develop menu."; log(config, "error", errorMessage); contextReport = { - result: { status: "SKIPPED", description: errorMessage }, + result: "SKIPPED", + resultDescription: errorMessage, ...contextReport, }; report.summary.contexts.skipped++; diff --git a/test/core.test.js b/test/core.test.js index d18a119..953d85b 100644 --- a/test/core.test.js +++ b/test/core.test.js @@ -110,4 +110,62 @@ describe("Run tests successfully", function () { fs.unlinkSync(tempFilePath); } }); + + it("Test is marked as skipped when all contexts are skipped", async () => { + // Create a spec with a context for a different platform than the current one. + // The resolver will generate a context that doesn't match the current platform, + // which will cause it to be skipped. + const currentPlatform = require("os").platform(); + const targetPlatform = + currentPlatform === "win32" ? "linux" : "windows"; + + const allContextsSkippedTest = { + id: "test-all-contexts-skipped", + contexts: [ + { + app: { name: "firefox" }, + platforms: [targetPlatform], // Will be skipped on current platform + }, + ], + tests: [ + { + id: "test-1", + steps: [ + { + action: "runShell", + command: "echo 'This should not run'", + }, + ], + }, + ], + }; + + // Write the test to a temporary file + const tempFilePath = path.resolve("./test/temp-all-contexts-skipped.json"); + fs.writeFileSync( + tempFilePath, + JSON.stringify(allContextsSkippedTest, null, 2) + ); + const config = { + input: tempFilePath, + logLevel: "silent", + }; + let result; + try { + result = await runTests(config); + // Verify that the test is marked as skipped, not passed + assert.equal(result.summary.tests.skipped, 1); + assert.equal(result.summary.tests.pass, 0); + assert.equal(result.summary.specs.skipped, 1); + assert.equal(result.summary.specs.pass, 0); + assert.equal(result.summary.contexts.skipped, 1); + // Also verify the actual test result + assert.equal(result.specs[0].result, "SKIPPED"); + assert.equal(result.specs[0].tests[0].result, "SKIPPED"); + assert.equal(result.specs[0].tests[0].contexts[0].result, "SKIPPED"); + } finally { + // Ensure cleanup even on failure + fs.unlinkSync(tempFilePath); + } + }); });