Fix incorrect PASS status when all contexts are skipped

.gitignore

@@ -107,4 +107,7 @@ dist
 *.mp4
 
 # Browser snapshots
-browser-snapshots
+browser-snapshots
+
+# Test output artifacts
+test/artifacts/output/

package.json

@@ -29,7 +29,7 @@
   "dependencies": {
     "@apidevtools/json-schema-ref-parser": "^14.2.1",
     "@ffmpeg-installer/ffmpeg": "^1.1.0",
-    "@puppeteer/browsers": "^2.10.12",
+    "@puppeteer/browsers": "^2.10.13",
     "ajv": "^8.17.1",
     "appium": "^2.19.0",
     "appium-chromium-driver": "1.5.1",
@@ -44,7 +44,7 @@
     "json-schema-faker": "^0.5.9",
     "pixelmatch": "^5.3.0",
     "pngjs": "^7.0.0",
-    "posthog-node": "^5.10.4",
+    "posthog-node": "^5.11.0",
     "sharp": "^0.34.4",
     "tree-kill": "^1.2.2",
     "webdriverio": "8.45.0"

src/tests.js

@@ -505,7 +505,7 @@ async function runSpecs({ resolvedTests }) {
               context.platform
             }", "apps": ${JSON.stringify(context.apps)}}`
           );
-          contextReport = { result: { status: "SKIPPED" }, ...contextReport };
+          contextReport = { result: "SKIPPED", ...contextReport };
           report.summary.contexts.skipped++;
           testReport.contexts.push(contextReport);
           continue;
@@ -563,7 +563,8 @@ async function runSpecs({ resolvedTests }) {
                   " Make sure you've run `safaridriver --enable` in a terminal and enabled 'Allow Remote Automation' in Safari's Develop menu.";
               log(config, "error", errorMessage);
               contextReport = {
-                result: { status: "SKIPPED", description: errorMessage },
+                result: "SKIPPED",
+                resultDescription: errorMessage,
                 ...contextReport,
               };
               report.summary.contexts.skipped++;

test/core.test.js

@@ -110,4 +110,62 @@ describe("Run tests successfully", function () {
       fs.unlinkSync(tempFilePath);
     }
   });
+
+  it("Test is marked as skipped when all contexts are skipped", async () => {
+    // Create a spec with a context for a different platform than the current one.
+    // The resolver will generate a context that doesn't match the current platform,
+    // which will cause it to be skipped.
+    const currentPlatform = require("os").platform();
+    const targetPlatform =
+      currentPlatform === "win32" ? "linux" : "windows";
+
+    const allContextsSkippedTest = {
+      id: "test-all-contexts-skipped",
+      contexts: [
+        {
+          app: { name: "firefox" },
+          platforms: [targetPlatform], // Will be skipped on current platform
+        },
+      ],
+      tests: [
+        {
+          id: "test-1",
+          steps: [
+            {
+              action: "runShell",
+              command: "echo 'This should not run'",
+            },
+          ],
+        },
+      ],
+    };
+
+    // Write the test to a temporary file
+    const tempFilePath = path.resolve("./test/temp-all-contexts-skipped.json");
+    fs.writeFileSync(
+      tempFilePath,
+      JSON.stringify(allContextsSkippedTest, null, 2)
+    );
+    const config = {
+      input: tempFilePath,
+      logLevel: "silent",
+    };
+    let result;
+    try {
+      result = await runTests(config);
+      // Verify that the test is marked as skipped, not passed
+      assert.equal(result.summary.tests.skipped, 1);
+      assert.equal(result.summary.tests.pass, 0);
+      assert.equal(result.summary.specs.skipped, 1);
+      assert.equal(result.summary.specs.pass, 0);
+      assert.equal(result.summary.contexts.skipped, 1);
+      // Also verify the actual test result
+      assert.equal(result.specs[0].result, "SKIPPED");
+      assert.equal(result.specs[0].tests[0].result, "SKIPPED");
+      assert.equal(result.specs[0].tests[0].contexts[0].result, "SKIPPED");
+    } finally {
+      // Ensure cleanup even on failure
+      fs.unlinkSync(tempFilePath);
+    }
+  });
 });