v0.4.4.4 feat: deep mode (Step 3) — skeptical re-examination of ruled-out causes

VERSION

@@ -1 +1 @@
-0.4.4.3
+0.4.4.4

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dops-assistant",
-  "version": "0.4.4.3",
+  "version": "0.4.4.4",
   "description": "Agentic infrastructure monitoring assistant — Grafana MCP + CLI",
   "type": "module",
   "main": "dist/index.js",

src/config/schema.test.ts

@@ -41,6 +41,22 @@ describe("ConfigSchema – defaults", () => {
     }
   });
 
+  it("defaults deep mode (Step 3) to OFF — hidden from users until the orchestrator ships", () => {
+    const result = ConfigSchema.safeParse({
+      llm: { apiKey: "sk-test", model: "gpt-4" },
+      providers: [grafanaProvider],
+    });
+    expect(result.success).toBe(true);
+    if (result.success) {
+      // Master gate must default false: the bounded deep mode only re-judges
+      // the existing RCA; the "investigate for the real cause" version is the
+      // not-yet-built Autonomous Orchestrator. Shipping ON would expose a
+      // half-feature. Both deepModeEnabled and deepModeOnComplete stay off.
+      expect(result.data.agent.deepModeEnabled).toBe(false);
+      expect(result.data.agent.deepModeOnComplete).toBe(false);
+    }
+  });
+
   it("accepts serviceAliases config", () => {
     const result = ConfigSchema.safeParse({
       llm,

src/config/schema.ts

@@ -129,6 +129,24 @@ const AgentSchema = z.object({
    * Opt-in while the loop is validated against labeled incidents.
    */
   synthesisLoopRounds: z.number().int().min(1).max(5).default(1),
+  /**
+   * Deep mode (Step 3) "from start": when true, an interactive investigation
+   * that ran the loop and ruled causes out automatically chains the deep
+   * re-examination on completion — no second click. Requires synthesisLoopRounds
+   * > 1 to have anything to re-examine. Default off (deep mode stays on-demand).
+   */
+  deepModeOnComplete: z.boolean().default(false),
+  /**
+   * Master switch for deep mode (Step 3) user exposure. Default OFF: the
+   * bounded re-examination only re-judges the existing RCA's hypotheses
+   * (resurrect a dismissed cause / weaken the confirmed one) — it does NOT
+   * investigate freely for the real cause. That fuller capability is the
+   * Autonomous Orchestrator (designed, not yet built). Until it ships we keep
+   * deep mode hidden from users: the "Deep investigate" button is suppressed
+   * and `deep_mode_investigate` is rejected unless this is true. Flip to true
+   * for internal testing (e.g. dev/config.yaml).
+   */
+  deepModeEnabled: z.boolean().default(false),
   // @deprecated: use top-level `memory` config instead; will be removed in a future version
   conversationMemory: ConversationMemorySchema.optional().default({}),
   investigationTriggerPhrases: z.array(z.string()).optional().default([

src/server/agents.ts

@@ -17,7 +17,10 @@
 
 import { randomUUID } from "node:crypto";
 import type { ServiceConfig, DiscoveryConfig } from "../config/schema.js";
-import type { RcaReport } from "../types/rca-types.js";
+import type { RcaReport, DeepModeReport } from "../types/rca-types.js";
+import type { AgentStreamEvent } from "../types/ws-types.js";
+import { runDeepMode, buildReexamineTargets, widenTimeRange } from "../workflows/steps/deep-mode.js";
+import { createGatherEvidence } from "../workflows/steps/hypothesis-requery.js";
 import { createLogger } from "../logger.js";
 
 const logger = createLogger("mastra-chat");
@@ -602,6 +605,96 @@ export async function createMastraAdapters(deps: MastraAdapterDeps) {
 
   const investigationAgent = new MastraInvestigationAdapter(workflowConfig);
 
+  /**
+   * Deep mode (Step 3): re-examine a completed investigation's ruled-out
+   * hypotheses with deeper read-only re-queries, resurrecting any the loop
+   * dismissed on thin evidence. Reuses the investigation providers + model
+   * wired above (no duplication). Returns a serializable DeepModeReport the
+   * caller persists onto the stored RcaReport. Read-only throughout.
+   */
+  async function deepModeReexamine(
+    report: RcaReport,
+    opts?: { onStep?: (ev: Omit<AgentStreamEvent, "seq">) => void; maxReexamine?: number },
+  ): Promise<DeepModeReport> {
+    const step = opts?.onStep ?? (() => {});
+    const examinedAt = new Date().toISOString();
+    const maxReexamine = opts?.maxReexamine ?? 3;
+    // Resurrect ruled-out causes, or — when none were ruled out — skeptically
+    // re-test the loop's standing conclusion (refute the confirmed cause).
+    const targets = buildReexamineTargets(report.hypotheses ?? [], report.ruledOut ?? [], report.loopOutcome, maxReexamine);
+    if (targets.length === 0) {
+      return { reexamined: [], resurrected: [], shaken: [], outcome: "nothing-to-examine", examinedAt };
+    }
+    const mode = targets[0].priorStanding === "ruled-out" ? "resurrect" : "refute";
+    // Translate raw MCP tool names into plain English for the stream.
+    const friendlyTool = (t: string): string => {
+      const k = t.toLowerCase();
+      if (k.includes("event")) return "cluster events";
+      if (k.includes("prometheus") || k.includes("metric")) return "metrics";
+      if (k.includes("loki") || k.includes("log")) return "logs";
+      if (k.includes("pod")) return "pods";
+      if (k.includes("deployment")) return "deployments";
+      if (k.includes("datasource")) return "data sources";
+      return t.replace(/_/g, " ");
+    };
+    step(mode === "resurrect"
+      ? { verb: "reopening", target: `${targets.length} dismissed ${targets.length === 1 ? "cause" : "causes"}`, status: "running" }
+      : { verb: "double-checking", target: "the most likely cause", detail: "(nothing was ruled out, so re-testing what we confirmed)", status: "running" });
+    const timeRange = report.timeRange;
+    // Dig deeper than the loop did: re-query a BROADER window so precursors the
+    // narrow incident window missed can surface. The change-in-window predicate
+    // still anchors to the ORIGINAL incident onset (ctx.incidentTime), so a
+    // wider query window doesn't move what counts as "before the incident".
+    const deeperRange = widenTimeRange(timeRange);
+    const ctx = { incidentTime: timeRange?.from };
+    const gather = createGatherEvidence({
+      providers,
+      model: investigationModel,
+      timeRange: deeperRange,
+      useQuirkHandling: true,
+      onToolCall: (tool, _args, _result, _dur, error) =>
+        step({ verb: "looked at", target: friendlyTool(tool), targetKind: "query", status: error ? "rejected" : "done", indent: 1 }),
+      llmRetry: config.llm.retry,
+      ctx,
+    });
+    const result = await runDeepMode({
+      targets,
+      priorObservations: [],
+      maxReexamine,
+      gatherDeepEvidence: (h) => {
+        step({ verb: mode === "resurrect" ? "checking" : "re-checking", target: h.hypothesis, status: "running" });
+        return gather(h, 1);
+      },
+      ctx,
+    });
+    // Per-hypothesis verdicts are known only after the loop finishes.
+    for (const r of result.reexamined) {
+      if (r.priorStanding === "ruled-out") {
+        step(r.flipped
+          ? { verb: "Worth another look:", target: r.hypothesis, detail: "— deeper evidence now points to it", status: "strong" }
+          : { verb: "Still unlikely:", target: r.hypothesis, detail: "— deeper evidence still doesn't support it", status: "done" });
+      } else {
+        step(r.flipped
+          ? { verb: "Probably not the cause:", target: r.hypothesis, detail: "— the evidence that would confirm it isn't there", status: "rejected" }
+          : { verb: "Still the likely cause:", target: r.hypothesis, detail: "— deeper evidence backs it up", status: "strong" });
+      }
+    }
+    const toRef = (h: { hypothesis: string; prediction: unknown }) => ({ hypothesis: h.hypothesis, prediction: h.prediction as Record<string, unknown> });
+    return {
+      reexamined: result.reexamined.map((r) => ({
+        hypothesis: r.hypothesis,
+        priorStanding: r.priorStanding,
+        priorVerdict: r.priorVerdict,
+        deepVerdict: r.deepVerdict,
+        flipped: r.flipped,
+      })),
+      resurrected: result.resurrected.map(toRef),
+      shaken: result.shaken.map(toRef),
+      outcome: result.outcome,
+      examinedAt,
+    };
+  }
+
   const discoverAgent = deps.registryStore
     ? new MastraDiscoverAdapter({
         model: discoveryModel,
@@ -613,5 +706,5 @@ export async function createMastraAdapters(deps: MastraAdapterDeps) {
       })
     : undefined;
 
-  return { chatAgent, investigationAgent, discoverAgent };
+  return { chatAgent, investigationAgent, discoverAgent, deepModeReexamine };
 }

src/server/index.ts

@@ -657,10 +657,13 @@ async function main() {
   // depth — with validation above, this is theoretically unreachable.
   const basePathForScript = JSON.stringify(appBasePath).replace(/</g, "\\u003c");
   const demoModeActive = isDemoMode();
+  // Deep mode (Step 3) is hidden from users until the Autonomous Orchestrator
+  // ships; the web bundle reads this to suppress the "Deep investigate" button.
+  const deepModeEnabled = config.agent?.deepModeEnabled === true;
 
   function buildIndexHtml(): string {
     const raw = readFileSync(path.resolve(staticDir, "index.html"), "utf-8");
-    if (appBasePath === "/" && !demoModeActive) return raw;
+    if (appBasePath === "/" && !demoModeActive && !deepModeEnabled) return raw;
 
     // Rewrite any absolute /assets/... reference to ${base}assets/... when a
     // sub-path is configured.
@@ -676,6 +679,7 @@ async function main() {
     const globals: string[] = [];
     if (appBasePath !== "/") globals.push(`window.__APP_BASE__=${basePathForScript}`);
     if (demoModeActive) globals.push(`window.__DEMO_MODE__=true`);
+    if (deepModeEnabled) globals.push(`window.__DEEP_MODE_ENABLED__=true`);
     if (globals.length === 0) return afterAssets;
 
     const inlineScript = `<script>${globals.join(";")};</script>`;

src/server/llm-settings.ts

@@ -35,6 +35,13 @@ export interface StackLlmSettingsView {
     investigation?: ReasoningEffort;
     discovery?: ReasoningEffort;
   };
+  /** Synthesis hypothesis-loop rounds (config.agent.synthesisLoopRounds).
+   *  1 = single-pass (loop off); >1 = the rank→test→rule-out loop is on.
+   *  Read-only: this is file/Helm config, not a per-stack GUI setting. */
+  synthesisLoopRounds: number;
+  /** Deep mode (Step 3) "from start" — auto-chains deep re-examination after
+   *  interactive investigations (config.agent.deepModeOnComplete). Read-only. */
+  deepModeOnComplete: boolean;
 }
 
 const BUCKETS: ReasoningBucket[] = ["chat", "investigation", "discovery"];
@@ -82,5 +89,7 @@ export function getStackLlmSettingsView(
       investigation: cfg.investigation,
       discovery: cfg.discovery,
     },
+    synthesisLoopRounds: config.agent?.synthesisLoopRounds ?? 1,
+    deepModeOnComplete: config.agent?.deepModeOnComplete ?? false,
   };
 }

src/server/sanitize.ts

@@ -109,6 +109,14 @@ export const DeepInvestigateMessageSchema = z.object({
   message: boundedString(MAX_CHAT_MESSAGE_LENGTH),
 });
 
+// ── DeepModeInvestigateMessageSchema (Step 3) ───────────────────────────────
+// Deep mode re-examines a completed investigation's ruled-out hypotheses; it
+// carries no free-text, just the target investigation id.
+export const DeepModeInvestigateMessageSchema = z.object({
+  type: z.literal("deep_mode_investigate"),
+  investigationId: z.string().min(1).max(100),
+});
+
 // ── SkillInputSchema ────────────────────────────────────────────────────────
 
 const MAX_SKILL_TITLE_LENGTH = 500;