Autonomy tests added grid

Added three testing layers for FlowPilot autonomy:
- Unit tests for token tracking, soul prompt builder, and system prompt compilation (unit-tests.test.ts)
- Edge-function integration tests for agent-execute and flowpilot-heartbeat (integration-autonomy.test.ts)
- Scenario eval suite with seeded DB checks for heartbeat state, atomic checkout, memory isolation, token logging, and CMS schema counts (scenario-eval-suite.test.ts)

These tests cover deterministic functions, API endpoints, and end-to-end scenario benchmarks to validate claims and guard against race conditions during autonomous improvements.

X-Lovable-Edit-ID: edt-73b0757f-7a16-4a0b-bbe8-67b32b442946
Co-authored-by: magnusfroste <38864257+magnusfroste@users.noreply.github.com>

Author: lovable-dev[bot]

src/test/autonomy/unit-tests.test.ts

@@ -0,0 +1,277 @@
+/**
+ * Lager 1: Unit Tests — Autonomy Functions
+ * 
+ * Tests pure, deterministic functions from the autonomy layer:
+ * - buildSystemPrompt (prompt compiler)
+ * - extractTokenUsage / accumulateTokens / isOverBudget
+ * - buildSoulPrompt
+ * 
+ * These don't need DB or network — fast and reliable.
+ */
+import { describe, it, expect } from "vitest";
+
+// We re-implement the pure functions here since they live in Deno edge functions.
+// This mirrors the logic from agent-reason.ts for testability.
+
+// ─── Token Tracking ───────────────────────────────────────────────────────────
+
+interface TokenUsage {
+  prompt_tokens: number;
+  completion_tokens: number;
+  total_tokens: number;
+}
+
+function extractTokenUsage(aiData: any): TokenUsage {
+  const usage = aiData.usage || {};
+  return {
+    prompt_tokens: usage.prompt_tokens || 0,
+    completion_tokens: usage.completion_tokens || 0,
+    total_tokens: (usage.prompt_tokens || 0) + (usage.completion_tokens || 0),
+  };
+}
+
+function accumulateTokens(current: TokenUsage, incoming: TokenUsage): TokenUsage {
+  return {
+    prompt_tokens: current.prompt_tokens + incoming.prompt_tokens,
+    completion_tokens: current.completion_tokens + incoming.completion_tokens,
+    total_tokens: current.total_tokens + incoming.total_tokens,
+  };
+}
+
+function isOverBudget(usage: TokenUsage, budget: number): boolean {
+  return usage.total_tokens >= budget;
+}
+
+// ─── Soul Prompt Builder ──────────────────────────────────────────────────────
+
+function buildSoulPrompt(soul: any, identity: any): string {
+  let prompt = '';
+  if (identity.name || identity.role) {
+    prompt += `\n\nIDENTITY:\nName: ${identity.name || 'FlowPilot'}\nRole: ${identity.role || 'CMS operator'}`;
+    if (identity.capabilities?.length) prompt += `\nCapabilities: ${identity.capabilities.join(', ')}`;
+    if (identity.boundaries?.length) prompt += `\nBoundaries: ${identity.boundaries.join('; ')}`;
+  }
+  if (soul.purpose) prompt += `\n\nSOUL:\nPurpose: ${soul.purpose}`;
+  if (soul.values?.length) prompt += `\nValues: ${soul.values.join('; ')}`;
+  if (soul.tone) prompt += `\nTone: ${soul.tone}`;
+  if (soul.philosophy) prompt += `\nPhilosophy: ${soul.philosophy}`;
+  return prompt;
+}
+
+// ─── Prompt Compiler (simplified) ─────────────────────────────────────────────
+
+type PromptMode = 'operate' | 'heartbeat' | 'chat';
+
+interface PromptCompilerInput {
+  mode: PromptMode;
+  soulPrompt: string;
+  memoryContext: string;
+  objectiveContext: string;
+  activityContext?: string;
+  statsContext?: string;
+  automationContext?: string;
+  healingReport?: string;
+  cmsSchemaContext?: string;
+  heartbeatState?: string;
+  tokenBudget?: number;
+  maxIterations?: number;
+  chatSystemPrompt?: string;
+}
+
+function buildSystemPrompt(input: PromptCompilerInput): string {
+  const { mode, soulPrompt, memoryContext, objectiveContext } = input;
+
+  if (mode === 'chat' && input.chatSystemPrompt) {
+    return input.chatSystemPrompt;
+  }
+
+  const parts: string[] = [];
+
+  if (mode === 'heartbeat') {
+    parts.push(`You are FlowPilot running in AUTONOMOUS HEARTBEAT mode. No human is watching.`);
+  } else {
+    parts.push(`You are FlowPilot — an autonomous, self-improving AI agent that operates a CMS platform.`);
+  }
+
+  parts.push(soulPrompt);
+
+  if (input.cmsSchemaContext) {
+    parts.push(input.cmsSchemaContext);
+  }
+
+  // Core instructions (abbreviated for test)
+  parts.push('CORE_INSTRUCTIONS');
+
+  if (mode === 'heartbeat') {
+    parts.push(`\nCONTEXT:`);
+    parts.push(memoryContext);
+    parts.push(objectiveContext);
+    if (input.automationContext) parts.push(input.automationContext);
+    if (input.activityContext) parts.push(input.activityContext);
+    if (input.statsContext) parts.push(input.statsContext);
+    if (input.healingReport) parts.push(input.healingReport);
+    if (input.heartbeatState) parts.push(input.heartbeatState);
+    if (input.tokenBudget) {
+      parts.push(`\nTOKEN BUDGET: ${input.tokenBudget} tokens max. Be efficient — stop early if approaching the limit.`);
+    }
+    parts.push('');
+    parts.push('HEARTBEAT_PROTOCOL');
+    parts.push(`\n- Max ${input.maxIterations || 8} tool iterations per heartbeat`);
+  } else {
+    parts.push(memoryContext);
+    parts.push(`\nOBJECTIVES:\n- After executing skills that contribute to an objective, update progress.\n- When all success_criteria are met, mark as complete.`);
+    parts.push(objectiveContext);
+  }
+
+  return parts.filter(Boolean).join('\n');
+}
+
+// ═══════════════════════════════════════════════════════════════════════════════
+// TESTS
+// ═══════════════════════════════════════════════════════════════════════════════
+
+describe("Token Tracking", () => {
+  it("extracts token usage from AI response", () => {
+    const aiData = { usage: { prompt_tokens: 100, completion_tokens: 50 } };
+    const result = extractTokenUsage(aiData);
+    expect(result).toEqual({ prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 });
+  });
+
+  it("handles missing usage gracefully", () => {
+    expect(extractTokenUsage({})).toEqual({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 });
+    expect(extractTokenUsage({ usage: {} })).toEqual({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 });
+  });
+
+  it("accumulates tokens correctly across iterations", () => {
+    const a: TokenUsage = { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 };
+    const b: TokenUsage = { prompt_tokens: 200, completion_tokens: 80, total_tokens: 280 };
+    const result = accumulateTokens(a, b);
+    expect(result).toEqual({ prompt_tokens: 300, completion_tokens: 130, total_tokens: 430 });
+  });
+
+  it("accumulates from zero", () => {
+    const zero: TokenUsage = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 };
+    const incoming: TokenUsage = { prompt_tokens: 500, completion_tokens: 100, total_tokens: 600 };
+    expect(accumulateTokens(zero, incoming)).toEqual(incoming);
+  });
+
+  it("detects budget exceeded", () => {
+    expect(isOverBudget({ prompt_tokens: 40000, completion_tokens: 10000, total_tokens: 50000 }, 50000)).toBe(true);
+    expect(isOverBudget({ prompt_tokens: 40000, completion_tokens: 10001, total_tokens: 50001 }, 50000)).toBe(true);
+  });
+
+  it("allows within budget", () => {
+    expect(isOverBudget({ prompt_tokens: 30000, completion_tokens: 10000, total_tokens: 40000 }, 50000)).toBe(false);
+    expect(isOverBudget({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, 50000)).toBe(false);
+  });
+});
+
+describe("Soul Prompt Builder", () => {
+  it("builds prompt with full soul and identity", () => {
+    const soul = { purpose: "Help grow the business", values: ["Honesty", "Growth"], tone: "professional", philosophy: "Always add value" };
+    const identity = { name: "Aria", role: "Digital consultant", capabilities: ["SEO", "Content"], boundaries: ["No spam"] };
+    const prompt = buildSoulPrompt(soul, identity);
+
+    expect(prompt).toContain("Name: Aria");
+    expect(prompt).toContain("Role: Digital consultant");
+    expect(prompt).toContain("Capabilities: SEO, Content");
+    expect(prompt).toContain("Boundaries: No spam");
+    expect(prompt).toContain("Purpose: Help grow the business");
+    expect(prompt).toContain("Values: Honesty; Growth");
+    expect(prompt).toContain("Tone: professional");
+    expect(prompt).toContain("Philosophy: Always add value");
+  });
+
+  it("uses defaults for missing identity fields", () => {
+    const prompt = buildSoulPrompt({}, { name: null, role: null });
+    // No identity or soul sections when empty
+    expect(prompt).toBe('');
+  });
+
+  it("handles partial soul", () => {
+    const prompt = buildSoulPrompt({ purpose: "Grow traffic" }, {});
+    expect(prompt).toContain("Purpose: Grow traffic");
+    expect(prompt).not.toContain("IDENTITY");
+  });
+});
+
+describe("Prompt Compiler (buildSystemPrompt)", () => {
+  const baseInput: PromptCompilerInput = {
+    mode: 'operate',
+    soulPrompt: 'SOUL: Test',
+    memoryContext: 'MEMORY: user likes blue',
+    objectiveContext: 'OBJ: Grow traffic by 20%',
+  };
+
+  it("operates mode includes objectives section", () => {
+    const prompt = buildSystemPrompt(baseInput);
+    expect(prompt).toContain("autonomous, self-improving AI agent");
+    expect(prompt).toContain("SOUL: Test");
+    expect(prompt).toContain("MEMORY: user likes blue");
+    expect(prompt).toContain("OBJECTIVES:");
+    expect(prompt).toContain("OBJ: Grow traffic by 20%");
+    expect(prompt).not.toContain("HEARTBEAT");
+  });
+
+  it("heartbeat mode includes protocol and context", () => {
+    const prompt = buildSystemPrompt({
+      ...baseInput,
+      mode: 'heartbeat',
+      activityContext: 'Recent: blog post created',
+      statsContext: 'Views: 500',
+      automationContext: 'Cron: daily-report',
+      maxIterations: 5,
+    });
+    expect(prompt).toContain("AUTONOMOUS HEARTBEAT mode");
+    expect(prompt).toContain("CONTEXT:");
+    expect(prompt).toContain("Recent: blog post created");
+    expect(prompt).toContain("Views: 500");
+    expect(prompt).toContain("Cron: daily-report");
+    expect(prompt).toContain("HEARTBEAT_PROTOCOL");
+    expect(prompt).toContain("Max 5 tool iterations");
+  });
+
+  it("injects CMS schema when provided", () => {
+    const prompt = buildSystemPrompt({
+      ...baseInput,
+      cmsSchemaContext: 'CMS: 10 pages, 5 products, Stripe active',
+    });
+    expect(prompt).toContain("CMS: 10 pages, 5 products, Stripe active");
+  });
+
+  it("injects heartbeat state in heartbeat mode", () => {
+    const prompt = buildSystemPrompt({
+      ...baseInput,
+      mode: 'heartbeat',
+      heartbeatState: 'Last run: 2026-03-14, 3 objectives advanced',
+    });
+    expect(prompt).toContain("Last run: 2026-03-14, 3 objectives advanced");
+  });
+
+  it("includes token budget warning in heartbeat mode", () => {
+    const prompt = buildSystemPrompt({
+      ...baseInput,
+      mode: 'heartbeat',
+      tokenBudget: 50000,
+    });
+    expect(prompt).toContain("TOKEN BUDGET: 50000 tokens max");
+  });
+
+  it("chat mode uses chatSystemPrompt override", () => {
+    const prompt = buildSystemPrompt({
+      ...baseInput,
+      mode: 'chat',
+      chatSystemPrompt: 'You are a helpful assistant.',
+    });
+    expect(prompt).toBe('You are a helpful assistant.');
+  });
+
+  it("does not leak heartbeat state into operate mode", () => {
+    const prompt = buildSystemPrompt({
+      ...baseInput,
+      mode: 'operate',
+      heartbeatState: 'SHOULD NOT APPEAR',
+    });
+    expect(prompt).not.toContain("SHOULD NOT APPEAR");
+  });
+});

supabase/functions/tests/integration-autonomy.test.ts

@@ -0,0 +1,112 @@
+/**
+ * Lager 2: Edge Function Integration Tests — Autonomy
+ * 
+ * Tests real edge function endpoints with controlled inputs.
+ * Validates: response shapes, error handling, checkout logic, goal-aware execution.
+ * 
+ * Run with: supabase--test_edge_functions or Deno test
+ */
+import "https://deno.land/std@0.224.0/dotenv/load.ts";
+import { assertEquals, assertExists } from "https://deno.land/std@0.224.0/assert/mod.ts";
+
+const SUPABASE_URL = Deno.env.get("VITE_SUPABASE_URL") || Deno.env.get("SUPABASE_URL")!;
+const SUPABASE_ANON_KEY = Deno.env.get("VITE_SUPABASE_PUBLISHABLE_KEY") || Deno.env.get("SUPABASE_ANON_KEY")!;
+
+const headers = {
+  "Content-Type": "application/json",
+  "Authorization": `Bearer ${SUPABASE_ANON_KEY}`,
+};
+
+// ─── Helper ───────────────────────────────────────────────────────────────────
+
+async function callEdgeFunction(name: string, body: any): Promise<{ status: number; data: any }> {
+  const response = await fetch(`${SUPABASE_URL}/functions/v1/${name}`, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(body),
+  });
+  const text = await response.text();
+  let data;
+  try { data = JSON.parse(text); } catch { data = { raw: text }; }
+  return { status: response.status, data };
+}
+
+// ═══════════════════════════════════════════════════════════════════════════════
+// TESTS
+// ═══════════════════════════════════════════════════════════════════════════════
+
+Deno.test("agent-execute: rejects missing skill_name/skill_id", async () => {
+  const { status, data } = await callEdgeFunction("agent-execute", {
+    arguments: {},
+    agent_type: "flowpilot",
+  });
+  assertEquals(status, 400);
+  assertEquals(data.error, "skill_id or skill_name required");
+});
+
+Deno.test("agent-execute: returns 404 for nonexistent skill", async () => {
+  const { status, data } = await callEdgeFunction("agent-execute", {
+    skill_name: "nonexistent_skill_xyz_99",
+    arguments: {},
+    agent_type: "flowpilot",
+  });
+  assertEquals(status, 404);
+  assertExists(data.error);
+});
+
+Deno.test("agent-execute: accepts objective_context without error", async () => {
+  // This tests that the goal-aware execution path doesn't crash.
+  // Even if the skill doesn't exist, it should 404 not 500.
+  const { status } = await callEdgeFunction("agent-execute", {
+    skill_name: "nonexistent_skill_xyz_99",
+    arguments: { test: true },
+    agent_type: "flowpilot",
+    objective_context: {
+      goal: "Increase traffic by 20%",
+      step: "Step 1: Audit current SEO",
+      why: "Testing goal-aware execution",
+    },
+  });
+  // Should be 404 (skill not found), NOT 500 (crash)
+  assertEquals(status, 404);
+});
+
+Deno.test("agent-execute: blocks external scope from chat", async () => {
+  // This tests scope validation. We need a skill that's internal-only.
+  // Since we can't guarantee a specific skill exists, we just verify
+  // the endpoint responds correctly for valid input shapes.
+  const { status } = await callEdgeFunction("agent-execute", {
+    skill_name: "manage_site_settings",
+    arguments: { action: "get", section: "general" },
+    agent_type: "chat", // Chat trying to access internal skill
+  });
+  // Either 403 (blocked) or 404 (skill not found) are acceptable
+  const validStatuses = [403, 404];
+  assertEquals(validStatuses.includes(status), true, `Expected 403 or 404, got ${status}`);
+});
+
+Deno.test("flowpilot-heartbeat: responds with correct shape", async () => {
+  // Heartbeat might fail due to no AI key in test env, but should not 500 on CORS
+  const response = await fetch(`${SUPABASE_URL}/functions/v1/flowpilot-heartbeat`, {
+    method: "OPTIONS",
+    headers,
+  });
+  // OPTIONS should return 200 with CORS headers
+  const text = await response.text(); // Consume body
+  assertEquals(response.status, 200);
+  assertExists(response.headers.get("access-control-allow-origin"));
+});
+
+Deno.test("heartbeat: POST returns structured response or AI error", async () => {
+  const { status, data } = await callEdgeFunction("flowpilot-heartbeat", {});
+  // Either success (200 with status/actions/token_usage) or expected error (500 with message)
+  if (status === 200) {
+    assertExists(data.status);
+    assertExists(data.actions);
+    assertExists(data.token_usage);
+    assertEquals(typeof data.duration_ms, "number");
+  } else {
+    // AI provider not configured is expected in test environments
+    assertExists(data.error);
+  }
+});