fix(mechanic): isolate status file in tests + Force override for stuck runs (#100)

The real bug behind 'Run does nothing': the orchestration tests call the
real run_mechanic_cycle, which wrote progress to the PRODUCTION status
file (a test item 'Blow up mid-run', left non-terminal by the exception
test) — the run guard read that ghost and refused every click. Isolate
mechanic_status._STATUS_FILE per test so the suite never touches the file
the server's panel reads.

Also add an escape hatch: POST /api/mechanic/run?force=true overrides the
one-at-a-time guard (an agent is silent for minutes, so age can't prove a
run is dead). The panel offers 'force a new run?' on already_running.
mechanic.js -> ?v=4.

+tests: status isolated, force overrides the guard. Full suite 1801
passing; ruff clean.

Co-Authored-By: Claude <noreply@anthropic.com>

Author: rayketcham

src/project_forge/web/routes.py

@@ -1683,24 +1683,30 @@ async def api_mechanic_status():
 
 
 @router.post("/api/mechanic/run")
-async def api_mechanic_run(request: Request):
+async def api_mechanic_run(request: Request, force: bool = False):
     """Human-triggered single mechanic cycle (validation / on-demand). Launches
     a detached one-shot process so the server never blocks on the agent; the
-    resulting PR appears in the panel for review. Rate-limited."""
+    resulting PR appears in the panel for review. Rate-limited.
+
+    `force=true` overrides the one-at-a-time guard — for when a run looks stuck
+    (the agent is silent for minutes, so age alone can't prove it's dead)."""
     client_ip = request.client.host if request.client else "unknown"
     _check_rate_limit(f"mechanic-run:{client_ip}")
     from project_forge.cron.mechanic_runner import spawn_mechanic_run
     from project_forge.engine.mechanic_status import read_status, write_status
 
-    # Guard: one run at a time. A second concurrent run would double the
-    # subscription spend and race on the same branch.
+    # Guard: one run at a time (a second concurrent run doubles the spend and
+    # races on the branch) — unless the operator forces it.
     status = read_status()
-    if not status.get("terminal"):
-        return {"status": "already_running", "detail": status.get("message", "A mechanic run is already in progress.")}
+    if not force and not status.get("terminal"):
+        return {
+            "status": "already_running",
+            "detail": status.get("message", "A mechanic run is already in progress."),
+            "item": status.get("item", ""),
+        }
 
     # Write an immediate non-terminal status BEFORE spawning, so the panel's
-    # first poll sees progress instead of the previous run's stale/idle state
-    # (that race is what made 'Run now' look like nothing happened).
+    # first poll sees progress instead of the previous run's stale state.
     write_status("selecting")
     spawn_mechanic_run()
     return {"status": "started"}

src/project_forge/web/static/mechanic.js

@@ -104,20 +104,33 @@
         pollOnce();
     }
 
+    async function doRun(force) {
+        runBtn.disabled = true;
+        startedAt = Date.now();
+        setStatus('⏱ starting — this takes a few minutes (clone → Claude implements → full test suite → PR). Leave this open; the PR appears below when done.', 'loading');
+        try {
+            var resp = await fetch('/api/mechanic/run' + (force ? '?force=true' : ''), { method: 'POST', headers: headers() });
+            if (!resp.ok) throw new Error('HTTP ' + resp.status);
+            var data = await resp.json();
+            if (data.status === 'already_running') {
+                var item = data.item ? ' (item: ' + data.item + ')' : '';
+                if (confirm('A mechanic run is already in progress' + item + '. Agents run silently for several minutes, so it may still be working. Force a NEW run anyway?')) {
+                    return doRun(true);
+                }
+                startPolling();  // just show the in-progress run's live status
+                return;
+            }
+            startPolling();
+        } catch (e) {
+            setStatus('failed to start: ' + e.message, 'error');
+            runBtn.disabled = false;
+        }
+    }
+
     if (runBtn && statusEl) {
-        runBtn.addEventListener('click', async function () {
+        runBtn.addEventListener('click', function () {
             if (!confirm('Run one mechanic cycle now? It implements the top Think Tank item on your subscription and opens a PR here. Takes several minutes — leave the page open.')) return;
-            runBtn.disabled = true;
-            startedAt = Date.now();
-            setStatus('⏱ starting — this takes a few minutes (clone → Claude implements → full test suite → PR). Leave this open; the PR appears below when done.', 'loading');
-            try {
-                var resp = await fetch('/api/mechanic/run', { method: 'POST', headers: headers() });
-                if (!resp.ok) throw new Error('HTTP ' + resp.status);
-                startPolling();
-            } catch (e) {
-                setStatus('failed to start: ' + e.message, 'error');
-                runBtn.disabled = false;
-            }
+            doRun(false);
         });
     }
 

src/project_forge/web/templates/mechanic.html

@@ -56,5 +56,5 @@ <h1>&#128295; Forge Mechanic</h1>
     </div>
 </section>
 
-<script src="/static/mechanic.js?v=3"></script>
+<script src="/static/mechanic.js?v=4"></script>
 {% endblock %}

tests/test_mechanic_arm.py

@@ -127,3 +127,19 @@ async def test_run_endpoint_refuses_when_already_running(self, client):
             resp = await client.post("/api/mechanic/run")
         assert resp.json()["status"] == "already_running"
         sp.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_run_endpoint_force_overrides_guard(self, client):
+        """force=true starts a run even if one looks in-progress (the escape
+        hatch for a stuck run)."""
+        with (
+            patch(
+                "project_forge.engine.mechanic_status.read_status",
+                return_value={"terminal": False, "message": "busy"},
+            ),
+            patch("project_forge.engine.mechanic_status.write_status"),
+            patch("project_forge.cron.mechanic_runner.spawn_mechanic_run") as sp,
+        ):
+            resp = await client.post("/api/mechanic/run?force=true")
+        assert resp.json()["status"] == "started"
+        sp.assert_called_once()

tests/test_mechanic_engine.py

@@ -38,6 +38,16 @@ def _no_open_prs(monkeypatch):
     monkeypatch.setattr("project_forge.engine.mechanic_review.list_open_prs", lambda: [])
 
 
+@pytest.fixture(autouse=True)
+def _isolate_status(tmp_path, monkeypatch):
+    # run_mechanic_cycle writes progress via mechanic_status; isolate that file
+    # so tests NEVER touch the real one the running server's panel reads (a
+    # test's mid-run 'implementing' status was ghost-locking the run guard).
+    import project_forge.engine.mechanic_status as ms
+
+    monkeypatch.setattr(ms, "_STATUS_FILE", tmp_path / "mech-status.json")
+
+
 def _si(name: str, **over) -> Idea:
     base = dict(
         name=name,