desktop: detect no-progress loops + bounded repair ladder

Author: jbutler1980

packages/bytebot-agent/src/agent/agent.desktop-loop.fixture.spec.ts

@@ -0,0 +1,69 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+import { MessageContentType } from '@bytebot/shared';
+import {
+  DesktopLoopDetector,
+  buildDesktopActionSignature,
+} from './agent.desktop-safety';
+
+type FixtureRow = {
+  tool: string;
+  coordinates?: { x: number; y: number };
+  button?: string;
+  clickCount?: number;
+  screenshotHash: string;
+};
+
+describe('desktop loop detector fixture replay', () => {
+  it('replays the Google Flights modal stall fixture and trips quickly', () => {
+    const fixturePath = path.join(
+      __dirname,
+      'fixtures',
+      'desktop-loop-google-flights.jsonl',
+    );
+    const lines = fs
+      .readFileSync(fixturePath, 'utf8')
+      .split('\n')
+      .map((l) => l.trim())
+      .filter(Boolean);
+
+    const detector = new DesktopLoopDetector();
+
+    let lastResult: ReturnType<DesktopLoopDetector['record']> | null = null;
+    let steps = 0;
+
+    for (const line of lines) {
+      const row = JSON.parse(line) as FixtureRow;
+      steps++;
+
+      const block: any = {
+        type: MessageContentType.ToolUse,
+        id: `fx-${steps}`,
+        name: row.tool,
+        input: {},
+      };
+
+      if (row.coordinates) {
+        block.input.coordinates = row.coordinates;
+      }
+      if (row.tool === 'computer_click_mouse') {
+        block.input.button = row.button || 'left';
+        block.input.clickCount = row.clickCount || 1;
+      }
+
+      const signature = buildDesktopActionSignature(block);
+      lastResult = detector.record({
+        atMs: Date.now(),
+        signature,
+        screenshotHash: row.screenshotHash,
+      });
+
+      if (lastResult.interrupt) break;
+    }
+
+    expect(lastResult?.interrupt).toBe(true);
+    expect(lastResult?.rule).toBe('repeat_in_window_no_progress');
+    expect(steps).toBeLessThanOrEqual(12);
+  });
+});

packages/bytebot-agent/src/agent/agent.desktop-repair.spec.ts

@@ -0,0 +1,118 @@
+import {
+  decodePngDimensionsFromBase64,
+  parseUiRepairCandidate,
+  validateUiRepairCandidate,
+} from './agent.desktop-repair';
+
+describe('agent.desktop-repair', () => {
+  describe('parseUiRepairCandidate', () => {
+    it('parses valid JSON with rationale_code', () => {
+      const res = parseUiRepairCandidate(
+        JSON.stringify({
+          x: 10,
+          y: 20,
+          confidence: 0.9,
+          rationale_code: 'CLOSE_X',
+        }),
+      );
+      expect(res.ok).toBe(true);
+      if (res.ok) {
+        expect(res.candidate.rationaleCode).toBe('CLOSE_X');
+        expect(res.candidate.x).toBe(10);
+        expect(res.candidate.y).toBe(20);
+      }
+    });
+
+    it('rejects missing JSON', () => {
+      const res = parseUiRepairCandidate('nope');
+      expect(res.ok).toBe(false);
+    });
+
+    it('rejects invalid rationale_code', () => {
+      const res = parseUiRepairCandidate(
+        JSON.stringify({
+          x: 10,
+          y: 20,
+          confidence: 0.9,
+          rationale_code: 'DELETE_ALL',
+        }),
+      );
+      expect(res.ok).toBe(false);
+    });
+  });
+
+  describe('decodePngDimensionsFromBase64', () => {
+    it('extracts width/height from a tiny PNG', () => {
+      // 1x1 transparent PNG
+      const base64 =
+        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PF2K1QAAAABJRU5ErkJggg==';
+      const dims = decodePngDimensionsFromBase64(base64);
+      expect(dims).toEqual({ width: 1, height: 1 });
+    });
+  });
+
+  describe('validateUiRepairCandidate', () => {
+    it('accepts CLOSE_X in top-right region across multiple viewport sizes', () => {
+      const viewports = [
+        { width: 100, height: 100 },
+        { width: 800, height: 600 },
+        { width: 1366, height: 768 },
+        { width: 1920, height: 1080 },
+      ];
+
+      for (const vp of viewports) {
+        const res = validateUiRepairCandidate({
+          candidate: {
+            x: Math.floor(vp.width * 0.9),
+            y: Math.floor(vp.height * 0.1),
+            confidence: 0.9,
+            rationaleCode: 'CLOSE_X',
+          },
+          dimensions: vp,
+          minConfidence: 0.7,
+        });
+        expect(res.ok).toBe(true);
+      }
+    });
+
+    it('rejects CLOSE_X outside top-right region', () => {
+      const res = validateUiRepairCandidate({
+        candidate: { x: 10, y: 90, confidence: 0.9, rationaleCode: 'CLOSE_X' },
+        dimensions: { width: 100, height: 100 },
+        minConfidence: 0.7,
+      });
+      expect(res.ok).toBe(false);
+    });
+
+    it('accepts DISMISS_BUTTON only in conservative right-side region', () => {
+      const viewports = [
+        { width: 320, height: 240 },
+        { width: 1024, height: 768 },
+        { width: 1920, height: 1080 },
+      ];
+
+      for (const vp of viewports) {
+        const res = validateUiRepairCandidate({
+          candidate: {
+            x: Math.floor(vp.width * 0.85),
+            y: Math.floor(vp.height * 0.5),
+            confidence: 0.95,
+            rationaleCode: 'DISMISS_BUTTON',
+          },
+          dimensions: vp,
+          minConfidence: 0.7,
+        });
+        expect(res.ok).toBe(true);
+      }
+    });
+
+    it('rejects low confidence', () => {
+      const res = validateUiRepairCandidate({
+        candidate: { x: 90, y: 10, confidence: 0.2, rationaleCode: 'CLOSE_X' },
+        dimensions: { width: 100, height: 100 },
+        minConfidence: 0.7,
+      });
+      expect(res.ok).toBe(false);
+    });
+  });
+});