Add matrix filtering, retry, resume, cleanup, and HTML reports to harness

- Golem state cleanup uses golem server clean + restart between scenarios
- Interactive TTY confirmation when not running on CI
- --no-cleanup flag to skip cleanup
- HTML summary report generation

Author: Myestery

.github/workflows/skills-test.yaml

@@ -3,12 +3,10 @@ name: Skills Test Harness
 on:
   push:
     paths:
-      - 'skills/**'
-      - 'tests/harness/**'
+      - 'golem-skills/**'
   pull_request:
     paths:
-      - 'skills/**'
-      - 'tests/harness/**'
+      - 'golem-skills/**'
   schedule:
     - cron: '0 6 * * 1' # Weekly Monday 6am UTC
   workflow_dispatch:
@@ -30,17 +28,17 @@ jobs:
         with:
           node-version: 20
           cache: 'npm'
-          cache-dependency-path: tests/harness/package.json
+          cache-dependency-path: golem-skills/tests/harness/package.json
 
       - name: Install and Build
         run: |
-          cd tests/harness
+          cd golem-skills/tests/harness
           npm install
           npm run build
 
       - name: Run Unit Tests
         run: |
-          cd tests/harness
+          cd golem-skills/tests/harness
           npm test
 
   integration-test:
@@ -50,7 +48,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        agent: [claude-code, gemini, opencode]
+        agent: [claude-code, gemini, opencode, codex]
         language: [ts]
         exclude:
           - agent: claude-code
@@ -63,7 +61,7 @@ jobs:
         with:
           node-version: 20
           cache: 'npm'
-          cache-dependency-path: tests/harness/package.json
+          cache-dependency-path: golem-skills/tests/harness/package.json
 
       - name: Install Rust (with wasm32-wasip2)
         uses: dtolnay/rust-toolchain@stable
@@ -97,28 +95,54 @@ jobs:
 
       - name: Install Gemini CLI
         if: matrix.agent == 'gemini'
-        run: npm install -g @google/gemini-cli
+        run: |
+          npm install -g @google/gemini-cli
+          mkdir -p ~/.gemini
 
       - name: Install OpenCode
         if: matrix.agent == 'opencode'
         run: npm install -g opencode-ai
 
+      - name: Install Codex CLI
+        if: matrix.agent == 'codex'
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          npm install -g @openai/codex
+          printenv OPENAI_API_KEY | codex login --with-api-key
+
       - name: Install Harness Dependencies
         run: |
-          cd tests/harness
+          cd golem-skills/tests/harness
           npm install
+          npm run build
+          npm install -g tsx
+
+      - name: Setup isolated test directory
+        run: |
+          mkdir -p /tmp/harness-run
+          cp -r golem-skills/tests/harness/dist /tmp/harness-run/dist
+          cp -r golem-skills/tests/harness/node_modules /tmp/harness-run/node_modules
+          cp -r golem-skills/tests/harness/src /tmp/harness-run/src
+          cp -r golem-skills/tests/harness/scenarios /tmp/harness-run/scenarios
+          cp golem-skills/tests/harness/package.json /tmp/harness-run/
+          cp golem-skills/tests/harness/tsconfig.json /tmp/harness-run/
+          cp -r golem-skills/skills /tmp/harness-run/skills
+          cd /tmp/harness-run && git init -b main && git config user.email "ci@golem.cloud" && git config user.name "CI" && git add -A && git commit -m "init"
 
       - name: Run Skill Tests
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
-          cd tests/harness
-          npx tsx src/run.ts --agent ${{ matrix.agent }} --language ${{ matrix.language }} --scenarios scenarios
+          cd /tmp/harness-run
+          tsx src/run.ts --agent ${{ matrix.agent }} --language ${{ matrix.language }} --scenarios scenarios --skills ./skills
 
       - name: Upload Results
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: results-${{ matrix.agent }}-${{ matrix.language }}
-          path: tests/harness/results/
+          path: /tmp/harness-run/results/

.gitignore

@@ -21,6 +21,8 @@ logs
 data
 sdks/**/node_modules/
 sdks/**/dist/
+golem-skills/**/node_modules/
+golem-skills/**/dist/
 .DS_Store
 
 # locally kept custom dev files

AGENTS.md

@@ -99,5 +99,6 @@ All crate dependencies must have their versions specified in the root workspace
 - `golem-rib/` - Rib language implementation
 - `cli/` - CLI tools (golem-cli, golem)
 - `sdks/` - Language-specific SDKs (Rust, TypeScript) - **not part of main build flow, see SDK-specific AGENTS.md**
+- `golem-skills/` - Skill definitions and skill testing harness
 - `integration-tests/` - Integration test suite
 - `test-components/` - Test WASM components

skills/golem-new-project/SKILL.md …skills/skills/golem-new-project/SKILL.mdskills/golem-new-project/SKILL.md renamed to golem-skills/skills/golem-new-project/SKILL.md skills/golem-new-project/SKILL.md renamed to golem-skills/skills/golem-new-project/SKILL.md

@@ -5,7 +5,10 @@ description: "Creating a new Golem application project. Use when scaffolding a n
 
 # Creating a New Golem Project
 
-Use the `golem new` CLI command to scaffold a new Golem application with the desired language template.
+**Important: Do not try to build golem from scratch or install it manually.**
+
+Assume the `golem` or `golem-cli` binary exists and is added to PATH.
+Try `golem --version` to check if it exists. If not, try `golem-cli --version`. Every command below works for `golem` and `golem-cli`
 
 ## Step 1: Run `golem new`
 
@@ -39,6 +42,7 @@ After running `golem new`, verify the following:
 ## Step 3: Build the Project
 
 ```shell
+cd <APPLICATION_NAME>
 golem build
 ```
 ## Checklist

tests/harness/.gitignore golem-skills/tests/harness/.gitignoretests/harness/.gitignore renamed to golem-skills/tests/harness/.gitignore

@@ -0,0 +1,20 @@
+name: "golem-new-project-ts"
+settings:
+  timeout_per_subprompt: 300
+steps:
+  - id: "create-project"
+    prompt: "Create a new Golem application called test-app with TypeScript"
+    expectedSkills:
+      - "golem-new-project"
+    verify:
+      build: true
+
+  - id: "check-project-files"
+    shell:
+      command: "ls"
+      args: ["test-app/golem.yaml"]
+    expect:
+      exit_code: 0
+
+  # Deploy requires components to be added to the project first;
+  # golem-new-project only scaffolds the app structure.

tests/harness/package-lock.json …m-skills/tests/harness/package-lock.jsontests/harness/package-lock.json renamed to golem-skills/tests/harness/package-lock.json tests/harness/package-lock.json renamed to golem-skills/tests/harness/package-lock.json

@@ -1,5 +1,5 @@
-import { JSONPath } from 'jsonpath-plus';
-import { z } from 'zod';
+import { JSONPath } from "jsonpath-plus";
+import { z } from "zod";
 
 const ResultJsonAssertionSchema = z.object({
   path: z.string(),
@@ -35,23 +35,27 @@ export interface AssertionResult {
   message: string;
 }
 
-export function evaluate(context: AssertionContext, expect: ExpectSpec): AssertionResult[] {
+export function evaluate(
+  context: AssertionContext,
+  expect: ExpectSpec,
+): AssertionResult[] {
   const results: AssertionResult[] = [];
 
   if (expect.exit_code !== undefined) {
     results.push({
-      assertion: 'exit_code',
+      assertion: "exit_code",
       passed: context.exitCode === expect.exit_code,
-      message: context.exitCode === expect.exit_code
-        ? `exit code is ${expect.exit_code}`
-        : `expected exit code ${expect.exit_code}, got ${context.exitCode}`,
+      message:
+        context.exitCode === expect.exit_code
+          ? `exit code is ${expect.exit_code}`
+          : `expected exit code ${expect.exit_code}, got ${context.exitCode}`,
     });
   }
 
   if (expect.stdout_contains !== undefined) {
     const passed = context.stdout.includes(expect.stdout_contains);
     results.push({
-      assertion: 'stdout_contains',
+      assertion: "stdout_contains",
       passed,
       message: passed
         ? `stdout contains "${expect.stdout_contains}"`
@@ -62,7 +66,7 @@ export function evaluate(context: AssertionContext, expect: ExpectSpec): Asserti
   if (expect.stdout_not_contains !== undefined) {
     const passed = !context.stdout.includes(expect.stdout_not_contains);
     results.push({
-      assertion: 'stdout_not_contains',
+      assertion: "stdout_not_contains",
       passed,
       message: passed
         ? `stdout does not contain "${expect.stdout_not_contains}"`
@@ -74,7 +78,7 @@ export function evaluate(context: AssertionContext, expect: ExpectSpec): Asserti
     const regex = new RegExp(expect.stdout_matches);
     const passed = regex.test(context.stdout);
     results.push({
-      assertion: 'stdout_matches',
+      assertion: "stdout_matches",
       passed,
       message: passed
         ? `stdout matches /${expect.stdout_matches}/`
@@ -85,7 +89,7 @@ export function evaluate(context: AssertionContext, expect: ExpectSpec): Asserti
   if (expect.status !== undefined) {
     const passed = context.status === expect.status;
     results.push({
-      assertion: 'status',
+      assertion: "status",
       passed,
       message: passed
         ? `status is ${expect.status}`
@@ -94,10 +98,10 @@ export function evaluate(context: AssertionContext, expect: ExpectSpec): Asserti
   }
 
   if (expect.body_contains !== undefined) {
-    const body = context.body ?? '';
+    const body = context.body ?? "";
     const passed = body.includes(expect.body_contains);
     results.push({
-      assertion: 'body_contains',
+      assertion: "body_contains",
       passed,
       message: passed
         ? `body contains "${expect.body_contains}"`
@@ -106,11 +110,11 @@ export function evaluate(context: AssertionContext, expect: ExpectSpec): Asserti
   }
 
   if (expect.body_matches !== undefined) {
-    const body = context.body ?? '';
+    const body = context.body ?? "";
     const regex = new RegExp(expect.body_matches);
     const passed = regex.test(body);
     results.push({
-      assertion: 'body_matches',
+      assertion: "body_matches",
       passed,
       message: passed
         ? `body matches /${expect.body_matches}/`
@@ -120,10 +124,15 @@ export function evaluate(context: AssertionContext, expect: ExpectSpec): Asserti
 
   if (expect.result_json && expect.result_json.length > 0) {
     for (const jsonAssert of expect.result_json) {
-      const pathResults = JSONPath({ path: jsonAssert.path, json: context.resultJson as object });
+      const pathResults = JSONPath({
+        path: jsonAssert.path,
+        json: context.resultJson as object,
+      });
 
       if (jsonAssert.equals !== undefined) {
-        const passed = pathResults.length > 0 && JSON.stringify(pathResults[0]) === JSON.stringify(jsonAssert.equals);
+        const passed =
+          pathResults.length > 0 &&
+          JSON.stringify(pathResults[0]) === JSON.stringify(jsonAssert.equals);
         results.push({
           assertion: `result_json[${jsonAssert.path}].equals`,
           passed,
@@ -134,7 +143,7 @@ export function evaluate(context: AssertionContext, expect: ExpectSpec): Asserti
       }
 
       if (jsonAssert.contains !== undefined) {
-        const value = pathResults.length > 0 ? String(pathResults[0]) : '';
+        const value = pathResults.length > 0 ? String(pathResults[0]) : "";
         const passed = value.includes(jsonAssert.contains);
         results.push({
           assertion: `result_json[${jsonAssert.path}].contains`,