Add example projects and update scoring evaluation with n=57 dataset

- Add A* pathfinding examples (Python + TypeScript) with test suites
- Add ML regression (linear regression) and classification (KNN) examples
- Update scoring-evaluation.md: expanded from n=21 to n=57 usable runs
  across 5 task types, 2 languages, 4 codebases
- Add statistical tests: Cochran's Q (p<0.001), Wilcoxon signed-rank,
  Cliff's delta (small effect d=0.183), Spearman rank correlation
- Add scoring-analysis.py script for reproducible statistical analysis
- Key finding: Copeland-Borda agree 81%, Weighted disagrees ~32%,
  agreement rates differ significantly across method pairs

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: unknown

docs/scoring-evaluation.md

@@ -0,0 +1,46 @@
+"""
+Grid-based pathfinding challenge for thinktank ensemble testing.
+
+The grid is a 2D list where:
+  0 = walkable
+  1 = obstacle
+
+Implement find_path() to find the shortest path from start to end
+using A* or any optimal pathfinding algorithm.
+
+Movement: 4-directional (up, down, left, right). No diagonals.
+"""
+
+from typing import Optional
+from dataclasses import dataclass
+
+Point = tuple[int, int]
+
+
+@dataclass
+class PathResult:
+    path: list[Point]
+    nodes_explored: int
+
+
+def find_path(
+    grid: list[list[int]],
+    start: Point,
+    end: Point,
+) -> Optional[PathResult]:
+    """
+    Find the shortest path from start to end on the grid.
+
+    YOUR TASK: Implement this function using A* pathfinding.
+    Choose your own heuristic, data structures, and optimizations.
+
+    Args:
+        grid: 2D grid (0=walkable, 1=obstacle)
+        start: Starting position (row, col)
+        end: Target position (row, col)
+
+    Returns:
+        PathResult with the shortest path and nodes explored, or None if unreachable
+    """
+    # TODO: Implement A* pathfinding
+    raise NotImplementedError("This is your task!")

examples/astar-python/grid.py

@@ -0,0 +1,2 @@
+#!/bin/bash
+cd "$(dirname "$0")" && python -m pytest test_pathfinding.py -v

examples/astar-python/run-tests.sh

@@ -0,0 +1,109 @@
+"""Tests for A* pathfinding implementation."""
+
+import time
+import unittest
+from grid import find_path, Point
+
+
+def is_valid_path(grid: list[list[int]], path: list[Point]) -> bool:
+    """Verify path is valid: each step is adjacent and on walkable terrain."""
+    for i, (r, c) in enumerate(path):
+        if r < 0 or r >= len(grid) or c < 0 or c >= len(grid[0]):
+            return False
+        if grid[r][c] != 0:
+            return False
+        if i > 0:
+            pr, pc = path[i - 1]
+            if abs(r - pr) + abs(c - pc) != 1:
+                return False
+    return True
+
+
+class TestPathfinding(unittest.TestCase):
+
+    def test_straight_line(self):
+        grid = [
+            [0, 0, 0, 0, 0],
+            [0, 0, 0, 0, 0],
+            [0, 0, 0, 0, 0],
+        ]
+        result = find_path(grid, (0, 0), (0, 4))
+        self.assertIsNotNone(result)
+        self.assertEqual(result.path[0], (0, 0))
+        self.assertEqual(result.path[-1], (0, 4))
+        self.assertEqual(len(result.path), 5)
+        self.assertTrue(is_valid_path(grid, result.path))
+
+    def test_around_obstacles(self):
+        grid = [
+            [0, 0, 0, 0, 0],
+            [0, 1, 1, 1, 0],
+            [0, 0, 0, 0, 0],
+        ]
+        result = find_path(grid, (0, 0), (2, 4))
+        self.assertIsNotNone(result)
+        self.assertEqual(result.path[0], (0, 0))
+        self.assertEqual(result.path[-1], (2, 4))
+        self.assertTrue(is_valid_path(grid, result.path))
+        self.assertEqual(len(result.path), 7)
+
+    def test_unreachable(self):
+        grid = [
+            [0, 1, 0],
+            [0, 1, 0],
+            [0, 1, 0],
+        ]
+        result = find_path(grid, (0, 0), (0, 2))
+        self.assertIsNone(result)
+
+    def test_start_equals_end(self):
+        grid = [[0, 0], [0, 0]]
+        result = find_path(grid, (1, 1), (1, 1))
+        self.assertIsNotNone(result)
+        self.assertEqual(len(result.path), 1)
+        self.assertEqual(result.path[0], (1, 1))
+
+    def test_maze(self):
+        grid = [
+            [0, 1, 0, 0, 0],
+            [0, 1, 0, 1, 0],
+            [0, 0, 0, 1, 0],
+            [1, 1, 0, 0, 0],
+            [0, 0, 0, 1, 0],
+        ]
+        result = find_path(grid, (0, 0), (4, 4))
+        self.assertIsNotNone(result)
+        self.assertEqual(result.path[0], (0, 0))
+        self.assertEqual(result.path[-1], (4, 4))
+        self.assertTrue(is_valid_path(grid, result.path))
+        self.assertEqual(len(result.path), 13)
+
+    def test_large_grid_performance(self):
+        size = 50
+        grid = [[0] * size for _ in range(size)]
+        for i in range(1, size - 1):
+            grid[i][size // 2] = 1
+
+        start = time.perf_counter()
+        result = find_path(grid, (0, 0), (size - 1, size - 1))
+        elapsed = time.perf_counter() - start
+
+        self.assertIsNotNone(result)
+        self.assertTrue(is_valid_path(grid, result.path))
+        self.assertLess(elapsed, 1.0, f"Should complete in < 1 second (took {elapsed:.3f}s)")
+        self.assertLess(result.nodes_explored, size * size)
+
+    def test_nodes_explored(self):
+        grid = [
+            [0, 0, 0],
+            [0, 0, 0],
+            [0, 0, 0],
+        ]
+        result = find_path(grid, (0, 0), (2, 2))
+        self.assertIsNotNone(result)
+        self.assertGreater(result.nodes_explored, 0)
+        self.assertLessEqual(result.nodes_explored, 9)
+
+
+if __name__ == "__main__":
+    unittest.main()

examples/astar-python/test_pathfinding.py

@@ -0,0 +1,2 @@
+node_modules/
+dist/

examples/astar/.gitignore

@@ -0,0 +1,15 @@
+{
+  "name": "astar-showcase",
+  "version": "1.0.0",
+  "description": "A* pathfinding showcase for thinktank ensemble testing",
+  "type": "module",
+  "scripts": {
+    "test": "npx tsx --test tests/*.test.ts",
+    "build": "npx tsc"
+  },
+  "devDependencies": {
+    "@types/node": "^25.5.0",
+    "tsx": "^4.21.0",
+    "typescript": "^6.0.2"
+  }
+}

examples/astar/package.json

@@ -0,0 +1,2 @@
+#!/bin/bash
+cd "$(dirname "$0")" && npx tsx --test tests/pathfinding.test.ts

examples/astar/run-tests.sh

@@ -0,0 +1,43 @@
+/**
+ * Grid-based pathfinding challenge for thinktank ensemble testing.
+ *
+ * The grid is a 2D array where:
+ *   0 = walkable
+ *   1 = obstacle
+ *
+ * Implement findPath() to find the shortest path from start to end
+ * using A* or any optimal pathfinding algorithm.
+ *
+ * The function should return an array of [row, col] coordinates
+ * representing the path from start to end (inclusive), or null
+ * if no path exists.
+ *
+ * Movement: 4-directional (up, down, left, right). No diagonals.
+ */
+
+export type Point = [number, number];
+
+export interface PathResult {
+  path: Point[];
+  nodesExplored: number;
+}
+
+/**
+ * Find the shortest path from start to end on the grid.
+ *
+ * YOUR TASK: Implement this function using A* pathfinding.
+ * Choose your own heuristic, data structures, and optimizations.
+ *
+ * @param grid - 2D grid (0=walkable, 1=obstacle)
+ * @param start - Starting position [row, col]
+ * @param end - Target position [row, col]
+ * @returns PathResult with the shortest path and nodes explored, or null if unreachable
+ */
+export function findPath(
+  grid: number[][],
+  start: Point,
+  end: Point,
+): PathResult | null {
+  // TODO: Implement A* pathfinding
+  throw new Error("Not implemented — this is your task!");
+}

examples/astar/src/grid.ts

@@ -0,0 +1,116 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { findPath, type Point } from "../src/grid.js";
+
+// Helper: verify path is valid (each step is adjacent and on walkable terrain)
+function isValidPath(grid: number[][], path: Point[]): boolean {
+  for (let i = 0; i < path.length; i++) {
+    const [r, c] = path[i]!;
+    if (r < 0 || r >= grid.length || c < 0 || c >= grid[0]!.length) return false;
+    if (grid[r]![c] !== 0) return false;
+    if (i > 0) {
+      const [pr, pc] = path[i - 1]!;
+      const dr = Math.abs(r - pr);
+      const dc = Math.abs(c - pc);
+      if (dr + dc !== 1) return false; // must be 4-directional adjacent
+    }
+  }
+  return true;
+}
+
+describe("A* Pathfinding", () => {
+  it("finds a straight-line path", () => {
+    const grid = [
+      [0, 0, 0, 0, 0],
+      [0, 0, 0, 0, 0],
+      [0, 0, 0, 0, 0],
+    ];
+    const result = findPath(grid, [0, 0], [0, 4]);
+    assert.ok(result, "should find a path");
+    assert.deepEqual(result.path[0], [0, 0], "starts at start");
+    assert.deepEqual(result.path[result.path.length - 1], [0, 4], "ends at end");
+    assert.equal(result.path.length, 5, "shortest path is 5 cells");
+    assert.ok(isValidPath(grid, result.path), "path must be valid");
+  });
+
+  it("navigates around obstacles", () => {
+    const grid = [
+      [0, 0, 0, 0, 0],
+      [0, 1, 1, 1, 0],
+      [0, 0, 0, 0, 0],
+    ];
+    const result = findPath(grid, [0, 0], [2, 4]);
+    assert.ok(result, "should find a path");
+    assert.deepEqual(result.path[0], [0, 0]);
+    assert.deepEqual(result.path[result.path.length - 1], [2, 4]);
+    assert.ok(isValidPath(grid, result.path), "path must be valid");
+    assert.equal(result.path.length, 7, "shortest path around obstacle is 7");
+  });
+
+  it("returns null for unreachable target", () => {
+    const grid = [
+      [0, 1, 0],
+      [0, 1, 0],
+      [0, 1, 0],
+    ];
+    const result = findPath(grid, [0, 0], [0, 2]);
+    assert.equal(result, null, "should return null when target is unreachable");
+  });
+
+  it("handles start equals end", () => {
+    const grid = [[0, 0], [0, 0]];
+    const result = findPath(grid, [1, 1], [1, 1]);
+    assert.ok(result, "should find a path");
+    assert.equal(result.path.length, 1, "path is just the start/end point");
+    assert.deepEqual(result.path[0], [1, 1]);
+  });
+
+  it("solves a maze", () => {
+    const grid = [
+      [0, 1, 0, 0, 0],
+      [0, 1, 0, 1, 0],
+      [0, 0, 0, 1, 0],
+      [1, 1, 0, 0, 0],
+      [0, 0, 0, 1, 0],
+    ];
+    const result = findPath(grid, [0, 0], [4, 4]);
+    assert.ok(result, "should find a path through the maze");
+    assert.deepEqual(result.path[0], [0, 0]);
+    assert.deepEqual(result.path[result.path.length - 1], [4, 4]);
+    assert.ok(isValidPath(grid, result.path), "path must be valid");
+    assert.equal(result.path.length, 13, "shortest maze path is 13");
+  });
+
+  it("handles large grid efficiently", () => {
+    // 50x50 grid with a clear path
+    const size = 50;
+    const grid: number[][] = Array.from({ length: size }, () =>
+      Array.from({ length: size }, () => 0)
+    );
+    // Add some obstacles but leave a clear path
+    for (let i = 1; i < size - 1; i++) {
+      grid[i]![Math.floor(size / 2)] = 1; // vertical wall with gap at top and bottom
+    }
+
+    const start = performance.now();
+    const result = findPath(grid, [0, 0], [size - 1, size - 1]);
+    const elapsed = performance.now() - start;
+
+    assert.ok(result, "should find a path on large grid");
+    assert.ok(isValidPath(grid, result.path), "path must be valid");
+    assert.ok(elapsed < 1000, `should complete in < 1 second (took ${elapsed.toFixed(0)}ms)`);
+    assert.ok(result.nodesExplored < size * size, "A* should not explore every cell");
+  });
+
+  it("tracks nodes explored accurately", () => {
+    const grid = [
+      [0, 0, 0],
+      [0, 0, 0],
+      [0, 0, 0],
+    ];
+    const result = findPath(grid, [0, 0], [2, 2]);
+    assert.ok(result, "should find a path");
+    assert.ok(result.nodesExplored > 0, "should explore at least 1 node");
+    assert.ok(result.nodesExplored <= 9, "should not explore more than grid size");
+  });
+});

examples/astar/tests/pathfinding.test.ts

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "target": "esnext",
+    "module": "nodenext",
+    "strict": true,
+    "outDir": "dist",
+    "rootDir": ".",
+    "lib": ["esnext"],
+    "types": ["node"]
+  },
+  "include": ["src/**/*", "tests/**/*"]
+}