diff --git a/bun.lock b/bun.lock index d44f3b87f9..cc3bdd32bd 100644 --- a/bun.lock +++ b/bun.lock @@ -61,6 +61,7 @@ "ollama-ai-provider-v2": "^1.5.4", "openai": "^6.9.1", "parse-duration": "^2.1.4", + "picomatch": "^4.0.3", "posthog-node": "^5.17.0", "quickjs-emscripten": "^0.31.0", "quickjs-emscripten-core": "^0.31.0", @@ -107,6 +108,7 @@ "@types/katex": "^0.16.7", "@types/markdown-it": "^14.1.2", "@types/minimist": "^1.2.5", + "@types/picomatch": "^4.0.2", "@types/react": "^18.2.0", "@types/react-dom": "^18.2.0", "@types/turndown": "^5.0.6", @@ -1395,6 +1397,8 @@ "@types/node": ["@types/node@20.19.25", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ=="], + "@types/picomatch": ["@types/picomatch@4.0.2", "", {}, "sha512-qHHxQ+P9PysNEGbALT8f8YOSHW0KJu6l2xU8DYY0fu/EmGxXdVnuTLvFUvBgPJMSqXq29SYHveejeAha+4AYgA=="], + "@types/plist": ["@types/plist@3.0.5", "", { "dependencies": { "@types/node": "*", "xmlbuilder": ">=11.0.1" } }, "sha512-E6OCaRmAe4WDmWNsL/9RMqdkkzDCY1etutkflWk4c+AcjDU07Pcz1fQwTX0TQz+Pxqn9i4L1TU3UFpjnrcDgxA=="], "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], diff --git a/docs/agent-skills.mdx b/docs/agent-skills.mdx index b614c171b3..d3a2ce3307 100644 --- a/docs/agent-skills.mdx +++ b/docs/agent-skills.mdx @@ -57,6 +57,7 @@ Optional fields: - `license` - `compatibility` - `metadata` (string key/value map) +- `include_files` (glob patterns, relative to the skill directory, for files to inline when the skill is loaded) Mux ignores unknown frontmatter keys (for example `allowed-tools`). @@ -69,6 +70,8 @@ description: Build and validate a release branch. license: MIT metadata: owner: platform +include_files: + - references/**/*.md --- # My Skill diff --git a/docs/agents.mdx b/docs/agents.mdx index 9e194a6498..daf258a940 100644 --- a/docs/agents.mdx +++ b/docs/agents.mdx @@ -76,6 +76,11 @@ name: My Agent # Display name in UI # Optional description: What this agent does # Shown in tooltips + +# Auto-include file context (workspace-root relative globs) +include_files: + - docs/**/*.md + - src/**/*.ts base: exec # Inherit from another agent (exec, plan, or custom agent id) # UI settings diff --git a/package.json b/package.json index 674f703417..bd3ff76ec7 100644 --- a/package.json +++ b/package.json @@ -101,6 +101,7 @@ "ollama-ai-provider-v2": "^1.5.4", "openai": "^6.9.1", "parse-duration": "^2.1.4", + "picomatch": "^4.0.3", "posthog-node": "^5.17.0", "quickjs-emscripten": "^0.31.0", "quickjs-emscripten-core": "^0.31.0", @@ -147,6 +148,7 @@ "@types/katex": "^0.16.7", "@types/markdown-it": "^14.1.2", "@types/minimist": "^1.2.5", + "@types/picomatch": "^4.0.2", "@types/react": "^18.2.0", "@types/react-dom": "^18.2.0", "@types/turndown": "^5.0.6", diff --git a/src/common/orpc/schemas.ts b/src/common/orpc/schemas.ts index 0ddfebd6c6..42dfc8f053 100644 --- a/src/common/orpc/schemas.ts +++ b/src/common/orpc/schemas.ts @@ -45,6 +45,9 @@ export { AgentSkillFrontmatterSchema, AgentSkillPackageSchema, AgentSkillScopeSchema, + IncludeFileGlobSchema, + IncludedFileSchema, + IncludeFilesContextSchema, SkillNameSchema, } from "./schemas/agentSkill"; diff --git a/src/common/orpc/schemas/agentDefinition.ts b/src/common/orpc/schemas/agentDefinition.ts index 95ef46a22f..b52e9cea54 100644 --- a/src/common/orpc/schemas/agentDefinition.ts +++ b/src/common/orpc/schemas/agentDefinition.ts @@ -1,5 +1,7 @@ import { z } from "zod"; +import { IncludeFileGlobSchema } from "./agentSkill"; + export const AgentDefinitionScopeSchema = z.enum(["built-in", "project", "global"]); // Agent IDs come from filenames (.md). @@ -46,7 +48,7 @@ const AgentDefinitionAiDefaultsSchema = z const AgentDefinitionPromptSchema = z .object({ - // When true, append this agent's body to the base agent's body (default: false = replace) + // When true, append this agent's body to the base agent's body (default: true); set false to replace. append: z.boolean().optional(), }) .strip(); @@ -67,6 +69,12 @@ export const AgentDefinitionFrontmatterSchema = z name: z.string().min(1).max(128), description: z.string().min(1).max(1024).optional(), + /** + * Glob patterns for files to automatically include in context when this agent runs. + * Patterns are relative to the workspace root (where tools run). + * Example: ["docs/**", "src/**"] + */ + include_files: z.array(IncludeFileGlobSchema).max(20).optional(), // Inheritance: reference a built-in or custom agent ID base: AgentIdSchema.optional(), diff --git a/src/common/orpc/schemas/agentSkill.ts b/src/common/orpc/schemas/agentSkill.ts index 81c629d049..6c0390426d 100644 --- a/src/common/orpc/schemas/agentSkill.ts +++ b/src/common/orpc/schemas/agentSkill.ts @@ -15,12 +15,35 @@ export const SkillNameSchema = z .max(64) .regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/); +/** + * Glob pattern for include_files. + * Patterns must be relative (no absolute paths, no ~, no ..). + */ +export const IncludeFileGlobSchema = z + .string() + .min(1) + .max(256) + .refine( + (pattern) => + !pattern.startsWith("/") && + !pattern.startsWith("~") && + !pattern.includes("..") && + !/^[A-Za-z]:[\\/]/.test(pattern), + { message: "Pattern must be relative (no absolute paths, ~, or ..)" } + ); + export const AgentSkillFrontmatterSchema = z.object({ name: SkillNameSchema, description: z.string().min(1).max(1024), license: z.string().optional(), compatibility: z.string().min(1).max(500).optional(), metadata: z.record(z.string(), z.string()).optional(), + /** + * Glob patterns for files to automatically include in context when the skill is read. + * Patterns are relative to the skill directory (where SKILL.md lives). + * Example: ["examples/*.ts", "schemas/**\/*.json"] + */ + include_files: z.array(IncludeFileGlobSchema).max(20).optional(), }); export const AgentSkillDescriptorSchema = z.object({ @@ -29,12 +52,39 @@ export const AgentSkillDescriptorSchema = z.object({ scope: AgentSkillScopeSchema, }); +/** + * Resolved file from include_files expansion. + */ +export const IncludedFileSchema = z.object({ + /** Path relative to skill directory */ + path: z.string(), + /** File content (may be truncated) */ + content: z.string(), + /** Whether content was truncated due to size/line limits */ + truncated: z.boolean(), +}); + +/** + * Context representation for files included via include_files. + * Rendered as XML using the `<@path>` tag format. + */ +export const IncludeFilesContextSchema = z.object({ + /** Successfully resolved files */ + files: z.array(IncludedFileSchema), + /** Patterns/files that had errors during resolution */ + errors: z.array(z.object({ pattern: z.string(), error: z.string() })), + /** Pre-rendered XML context (for direct injection) */ + rendered: z.string(), +}); + export const AgentSkillPackageSchema = z .object({ scope: AgentSkillScopeSchema, directoryName: SkillNameSchema, frontmatter: AgentSkillFrontmatterSchema, body: z.string(), + /** Resolved include_files context (present when frontmatter.include_files is set) */ + includeFilesContext: IncludeFilesContextSchema.optional(), }) .refine((value) => value.directoryName === value.frontmatter.name, { message: "SKILL.md frontmatter.name must match the parent directory name", diff --git a/src/node/services/agentDefinitions/agentDefinitionsService.test.ts b/src/node/services/agentDefinitions/agentDefinitionsService.test.ts index 482f5c6225..e19d1d49b2 100644 --- a/src/node/services/agentDefinitions/agentDefinitionsService.test.ts +++ b/src/node/services/agentDefinitions/agentDefinitionsService.test.ts @@ -10,6 +10,7 @@ import { discoverAgentDefinitions, readAgentDefinition, resolveAgentBody, + resolveAgentIncludeFilesPatterns, } from "./agentDefinitionsService"; async function writeAgent(root: string, id: string, name: string): Promise { @@ -229,4 +230,68 @@ Project body. expect(skippedPkg.scope).toBe("global"); expect(skippedPkg.frontmatter.name).toBe("Test Global"); }); + + test("resolveAgentIncludeFilesPatterns inherits by default, stops when prompt.append is false", async () => { + using tempDir = new DisposableTempDir("agent-include-files-patterns"); + const agentsRoot = path.join(tempDir.path, ".mux", "agents"); + await fs.mkdir(agentsRoot, { recursive: true }); + + await fs.writeFile( + path.join(agentsRoot, "base.md"), + `--- +name: Base +include_files: + - "README.md" + - "src/*.ts" +--- +Base instructions. +`, + "utf-8" + ); + + await fs.writeFile( + path.join(agentsRoot, "child.md"), + `--- +name: Child +base: base +include_files: + - "src/*.ts" + - "docs/**/*.md" +--- +Child instructions. +`, + "utf-8" + ); + + await fs.writeFile( + path.join(agentsRoot, "replacer.md"), + `--- +name: Replacer +base: base +prompt: + append: false +include_files: + - "only-child.txt" +--- +Replaced body. +`, + "utf-8" + ); + + const roots = { projectRoot: agentsRoot, globalRoot: agentsRoot }; + const runtime = new LocalRuntime(tempDir.path); + + const childPatterns = await resolveAgentIncludeFilesPatterns(runtime, tempDir.path, "child", { + roots, + }); + expect(childPatterns).toEqual(["README.md", "src/*.ts", "docs/**/*.md"]); + + const replacerPatterns = await resolveAgentIncludeFilesPatterns( + runtime, + tempDir.path, + "replacer", + { roots } + ); + expect(replacerPatterns).toEqual(["only-child.txt"]); + }); }); diff --git a/src/node/services/agentDefinitions/agentDefinitionsService.ts b/src/node/services/agentDefinitions/agentDefinitionsService.ts index 898ce48730..ea5064ae3b 100644 --- a/src/node/services/agentDefinitions/agentDefinitionsService.ts +++ b/src/node/services/agentDefinitions/agentDefinitionsService.ts @@ -471,3 +471,68 @@ export async function resolveAgentBody( return resolve(agentId, 0); } + +/** + * Resolve include_files patterns for an agent, including inherited patterns. + * + * Inheritance behavior mirrors resolveAgentBody: + * - If `prompt.append` is false, we do NOT inherit include_files from the base. + * - Otherwise, base patterns are prepended (base first, then child). + */ +export async function resolveAgentIncludeFilesPatterns( + runtime: Runtime, + workspacePath: string, + agentId: AgentId, + options?: { roots?: AgentDefinitionsRoots } +): Promise { + const visited = new Set(); + + async function resolve( + id: AgentId, + depth: number, + skipScopesAbove?: AgentDefinitionScope + ): Promise { + if (depth > MAX_INHERITANCE_DEPTH) { + throw new Error( + `Agent inheritance depth exceeded for '${id}' (max: ${MAX_INHERITANCE_DEPTH})` + ); + } + + const pkg = await readAgentDefinition(runtime, workspacePath, id, { + roots: options?.roots, + skipScopesAbove, + }); + + const visitKey = agentVisitKey(pkg.id, pkg.scope); + if (visited.has(visitKey)) { + throw new Error(`Circular agent inheritance detected: ${pkg.id} (${pkg.scope})`); + } + visited.add(visitKey); + + const baseId = pkg.frontmatter.base; + const shouldAppend = pkg.frontmatter.prompt?.append !== false; + + const patterns = pkg.frontmatter.include_files ?? []; + + if (!baseId || !shouldAppend) { + return patterns; + } + + const basePatterns = await resolve( + baseId, + depth + 1, + computeBaseSkipScope(baseId, id, pkg.scope) + ); + return [...basePatterns, ...patterns]; + } + + const patterns = await resolve(agentId, 0); + + // Preserve order while deduplicating. + const seen = new Set(); + return patterns.filter((pattern) => { + if (seen.has(pattern)) return false; + seen.add(pattern); + return true; + }); +} diff --git a/src/node/services/agentDefinitions/parseAgentDefinitionMarkdown.test.ts b/src/node/services/agentDefinitions/parseAgentDefinitionMarkdown.test.ts index 3d173b526b..7df3b1fbf8 100644 --- a/src/node/services/agentDefinitions/parseAgentDefinitionMarkdown.test.ts +++ b/src/node/services/agentDefinitions/parseAgentDefinitionMarkdown.test.ts @@ -73,6 +73,58 @@ Body ).toThrow(AgentDefinitionParseError); }); + test("parses include_files patterns", () => { + const content = `--- +name: With Files +include_files: + - "docs/**/*.md" + - "src/**/*.ts" +--- +Body +`; + + const result = parseAgentDefinitionMarkdown({ + content, + byteSize: Buffer.byteLength(content, "utf-8"), + }); + + expect(result.frontmatter.include_files).toEqual(["docs/**/*.md", "src/**/*.ts"]); + }); + + test("rejects absolute paths in include_files", () => { + const content = `--- +name: Bad Paths +include_files: + - "/etc/passwd" +--- +Body +`; + + expect(() => + parseAgentDefinitionMarkdown({ + content, + byteSize: Buffer.byteLength(content, "utf-8"), + }) + ).toThrow(AgentDefinitionParseError); + }); + + test("rejects path traversal in include_files", () => { + const content = `--- +name: Traversal +include_files: + - "../secret.txt" +--- +Body +`; + + expect(() => + parseAgentDefinitionMarkdown({ + content, + byteSize: Buffer.byteLength(content, "utf-8"), + }) + ).toThrow(AgentDefinitionParseError); + }); + test("parses tools as add/remove patterns", () => { const content = `--- name: Regex Tools diff --git a/src/node/services/agentIncludeFiles.ts b/src/node/services/agentIncludeFiles.ts new file mode 100644 index 0000000000..af124a3584 --- /dev/null +++ b/src/node/services/agentIncludeFiles.ts @@ -0,0 +1,65 @@ +import type { MuxMessage } from "@/common/types/message"; +import { createMuxMessage } from "@/common/types/message"; +import type { Runtime } from "@/node/runtime/Runtime"; +import { + renderIncludedFilesContext, + resolveIncludeFiles, +} from "@/node/services/agentSkills/includeFilesResolver"; + +/** + * Inject agent `include_files` context as a synthetic user message. + * + * This mirrors @file mention injection: we keep the system message stable (cache-friendly) + * while still giving the model immediate, structured file context. + */ +export async function injectAgentIncludeFiles( + messages: MuxMessage[], + options: { + runtime: Runtime; + workspacePath: string; + patterns: string[]; + abortSignal?: AbortSignal; + } +): Promise { + if (!Array.isArray(messages) || messages.length === 0) { + return messages; + } + + const patterns = options.patterns.filter(Boolean); + if (patterns.length === 0) { + return messages; + } + + // Find the last user-authored message (ignore synthetic injections like mode transitions). + let targetIndex = -1; + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + if (msg?.role === "user" && msg.metadata?.synthetic !== true) { + targetIndex = i; + break; + } + } + + if (targetIndex === -1) { + return messages; + } + + const resolved = await resolveIncludeFiles(options.runtime, options.workspacePath, patterns, { + abortSignal: options.abortSignal, + listMode: "git", + }); + + const rendered = renderIncludedFilesContext(resolved).trim(); + if (!rendered) { + return messages; + } + + const injected = createMuxMessage(`agent-include-files-${Date.now()}`, "user", rendered, { + timestamp: Date.now(), + synthetic: true, + }); + + const result = [...messages]; + result.splice(targetIndex, 0, injected); + return result; +} diff --git a/src/node/services/agentSkills/agentSkillsService.test.ts b/src/node/services/agentSkills/agentSkillsService.test.ts index 90e03384ef..12c59f81d0 100644 --- a/src/node/services/agentSkills/agentSkillsService.test.ts +++ b/src/node/services/agentSkills/agentSkillsService.test.ts @@ -8,16 +8,36 @@ import { LocalRuntime } from "@/node/runtime/LocalRuntime"; import { DisposableTempDir } from "@/node/services/tempDir"; import { discoverAgentSkills, readAgentSkill } from "./agentSkillsService"; -async function writeSkill(root: string, name: string, description: string): Promise { +interface WriteSkillOptions { + include_files?: string[]; + extraFiles?: Record; +} + +async function writeSkill( + root: string, + name: string, + description: string, + options?: WriteSkillOptions +): Promise { const skillDir = path.join(root, name); await fs.mkdir(skillDir, { recursive: true }); - const content = `--- -name: ${name} -description: ${description} ---- -Body -`; + + let frontmatter = `name: ${name}\ndescription: ${description}`; + if (options?.include_files) { + frontmatter += `\ninclude_files:\n${options.include_files.map((f) => ` - "${f}"`).join("\n")}`; + } + + const content = `---\n${frontmatter}\n---\nBody\n`; await fs.writeFile(path.join(skillDir, "SKILL.md"), content, "utf-8"); + + // Write extra files if specified + if (options?.extraFiles) { + for (const [filePath, fileContent] of Object.entries(options.extraFiles)) { + const fullPath = path.join(skillDir, filePath); + await fs.mkdir(path.dirname(fullPath), { recursive: true }); + await fs.writeFile(fullPath, fileContent, "utf-8"); + } + } } describe("agentSkillsService", () => { @@ -68,4 +88,62 @@ describe("agentSkillsService", () => { expect(resolved.package.scope).toBe("project"); expect(resolved.package.frontmatter.description).toBe("from project"); }); + + test("readAgentSkill resolves include_files patterns", async () => { + using project = new DisposableTempDir("agent-skills-include"); + + const projectSkillsRoot = path.join(project.path, ".mux", "skills"); + await writeSkill(projectSkillsRoot, "with-files", "skill with included files", { + include_files: ["examples/*.ts", "schemas/*.json"], + extraFiles: { + "examples/hello.ts": 'console.log("hello");', + "examples/world.ts": 'console.log("world");', + "schemas/config.json": '{"key": "value"}', + "other/ignored.txt": "should not be included", + }, + }); + + const roots = { projectRoot: projectSkillsRoot, globalRoot: "/nonexistent" }; + const runtime = new LocalRuntime(project.path); + + const name = SkillNameSchema.parse("with-files"); + const resolved = await readAgentSkill(runtime, project.path, name, { roots }); + + expect(resolved.package.includeFilesContext).toBeDefined(); + const ctx = resolved.package.includeFilesContext!; + + // Should have matched 3 files + expect(ctx.files.length).toBe(3); + + const paths = ctx.files.map((f) => f.path).sort(); + expect(paths).toEqual(["examples/hello.ts", "examples/world.ts", "schemas/config.json"]); + + // Check content is included + const hello = ctx.files.find((f) => f.path === "examples/hello.ts"); + expect(hello?.content).toBe('console.log("hello");'); + + // Check rendered XML uses <@path> format + expect(ctx.rendered).toContain("<@examples/hello.ts>"); + expect(ctx.rendered).toContain(""); + expect(ctx.rendered).toContain("```ts"); + expect(ctx.rendered).toContain('console.log("hello");'); + + // Should not include unmatched files + expect(ctx.rendered).not.toContain("ignored.txt"); + }); + + test("readAgentSkill without include_files has no context", async () => { + using project = new DisposableTempDir("agent-skills-no-include"); + + const projectSkillsRoot = path.join(project.path, ".mux", "skills"); + await writeSkill(projectSkillsRoot, "basic", "basic skill"); + + const roots = { projectRoot: projectSkillsRoot, globalRoot: "/nonexistent" }; + const runtime = new LocalRuntime(project.path); + + const name = SkillNameSchema.parse("basic"); + const resolved = await readAgentSkill(runtime, project.path, name, { roots }); + + expect(resolved.package.includeFilesContext).toBeUndefined(); + }); }); diff --git a/src/node/services/agentSkills/agentSkillsService.ts b/src/node/services/agentSkills/agentSkillsService.ts index 662562f07f..0a62061354 100644 --- a/src/node/services/agentSkills/agentSkillsService.ts +++ b/src/node/services/agentSkills/agentSkillsService.ts @@ -10,7 +10,10 @@ import { AgentSkillDescriptorSchema, AgentSkillPackageSchema, SkillNameSchema, + type IncludeFilesContextSchema, } from "@/common/orpc/schemas"; +import type { z } from "zod"; +import { resolveIncludeFiles, renderIncludedFilesContext } from "./includeFilesResolver"; import type { AgentSkillDescriptor, AgentSkillPackage, @@ -236,11 +239,23 @@ async function readAgentSkillFromDir( directoryName, }); + // Resolve include_files if specified + let includeFilesContext: z.infer | undefined; + if (parsed.frontmatter.include_files && parsed.frontmatter.include_files.length > 0) { + const resolved = await resolveIncludeFiles(runtime, skillDir, parsed.frontmatter.include_files); + includeFilesContext = { + files: resolved.files, + errors: resolved.errors, + rendered: renderIncludedFilesContext(resolved), + }; + } + const pkg: AgentSkillPackage = { scope, directoryName, frontmatter: parsed.frontmatter, body: parsed.body, + includeFilesContext, }; const validated = AgentSkillPackageSchema.safeParse(pkg); diff --git a/src/node/services/agentSkills/includeFilesResolver.test.ts b/src/node/services/agentSkills/includeFilesResolver.test.ts new file mode 100644 index 0000000000..543d70b363 --- /dev/null +++ b/src/node/services/agentSkills/includeFilesResolver.test.ts @@ -0,0 +1,188 @@ +import * as fs from "node:fs/promises"; +import { execSync } from "node:child_process"; +import * as path from "node:path"; + +import { describe, expect, test } from "bun:test"; + +import { LocalRuntime } from "@/node/runtime/LocalRuntime"; +import { DisposableTempDir } from "@/node/services/tempDir"; +import { + renderContextFile, + renderContextFileError, + resolveIncludeFiles, + renderIncludedFilesContext, +} from "./includeFilesResolver"; + +describe("includeFilesResolver", () => { + describe("renderContextFile", () => { + test("renders file with code fence and language detection", () => { + const result = renderContextFile({ + path: "example.ts", + content: 'const x = 1;\nconsole.log("hello");', + truncated: false, + }); + + expect(result).toBe( + '<@example.ts>\n```ts\nconst x = 1;\nconsole.log("hello");\n```\n' + ); + }); + + test("encodes paths for use in <@...> tag names", () => { + const result = renderContextFile({ + path: "docs/User Guide.md", + content: "hello", + truncated: false, + }); + + expect(result).toContain("<@docs/User%20Guide.md>"); + expect(result).toContain(""); + }); + + test("includes truncated attribute when content is truncated", () => { + const result = renderContextFile({ + path: "large.json", + content: '{"key": "value"}', + truncated: true, + }); + + expect(result).toContain('<@large.json truncated="true">'); + expect(result).toContain("```json"); + }); + + test("uses plain fence for unknown extensions", () => { + const result = renderContextFile({ + path: "unknown.xyz", + content: "some content", + truncated: false, + }); + + expect(result).toContain("```\n"); + expect(result).not.toContain("```xyz"); + }); + }); + + describe("renderContextFileError", () => { + test("renders error with escaped attributes", () => { + const result = renderContextFileError("bad/file.ts", 'Error with "quotes" & '); + expect(result).toBe( + '<@bad/file.ts error="Error with "quotes" & <brackets>" />' + ); + }); + }); + + describe("resolveIncludeFiles", () => { + test("matches files with glob patterns", async () => { + using tempDir = new DisposableTempDir("include-files-test"); + + // Create test files + await fs.mkdir(path.join(tempDir.path, "src"), { recursive: true }); + await fs.writeFile(path.join(tempDir.path, "src/a.ts"), "const a = 1;"); + await fs.writeFile(path.join(tempDir.path, "src/b.ts"), "const b = 2;"); + await fs.writeFile(path.join(tempDir.path, "src/c.js"), "const c = 3;"); + await fs.writeFile(path.join(tempDir.path, "README.md"), "# Readme"); + + const runtime = new LocalRuntime(tempDir.path); + const result = await resolveIncludeFiles(runtime, tempDir.path, ["src/*.ts"]); + + expect(result.errors).toHaveLength(0); + expect(result.files.length).toBe(2); + + const paths = result.files.map((f) => f.path).sort(); + expect(paths).toEqual(["src/a.ts", "src/b.ts"]); + }); + + test("handles multiple patterns", async () => { + using tempDir = new DisposableTempDir("include-files-multi"); + + await fs.mkdir(path.join(tempDir.path, "src"), { recursive: true }); + await fs.writeFile(path.join(tempDir.path, "src/main.ts"), "main"); + await fs.writeFile(path.join(tempDir.path, "README.md"), "readme"); + + const runtime = new LocalRuntime(tempDir.path); + const result = await resolveIncludeFiles(runtime, tempDir.path, ["src/*.ts", "*.md"]); + + expect(result.errors).toHaveLength(0); + expect(result.files.length).toBe(2); + + const paths = result.files.map((f) => f.path).sort(); + expect(paths).toEqual(["README.md", "src/main.ts"]); + }); + + test("supports listMode=git for deep paths", async () => { + using tempDir = new DisposableTempDir("include-files-git"); + execSync("git init -b main", { cwd: tempDir.path, stdio: "ignore" }); + + await fs.mkdir(path.join(tempDir.path, "a/b/c/d/e"), { recursive: true }); + await fs.writeFile(path.join(tempDir.path, "a/b/c/d/e/f.txt"), "deep"); + + const runtime = new LocalRuntime(tempDir.path); + const result = await resolveIncludeFiles(runtime, tempDir.path, ["a/b/c/d/e/f.txt"], { + listMode: "git", + }); + + expect(result.errors).toHaveLength(0); + expect(result.files.map((f) => f.path)).toEqual(["a/b/c/d/e/f.txt"]); + }); + + test("deduplicates files matched by multiple patterns", async () => { + using tempDir = new DisposableTempDir("include-files-dedup"); + + await fs.writeFile(path.join(tempDir.path, "file.ts"), "content"); + + const runtime = new LocalRuntime(tempDir.path); + const result = await resolveIncludeFiles(runtime, tempDir.path, ["*.ts", "file.*"]); + + expect(result.files.length).toBe(1); + expect(result.files[0]?.path).toBe("file.ts"); + }); + + test("skips binary files", async () => { + using tempDir = new DisposableTempDir("include-files-binary"); + + // Create a binary file with null bytes + await fs.writeFile(path.join(tempDir.path, "binary.dat"), Buffer.from([0x00, 0x01, 0x02])); + await fs.writeFile(path.join(tempDir.path, "text.txt"), "normal text"); + + const runtime = new LocalRuntime(tempDir.path); + const result = await resolveIncludeFiles(runtime, tempDir.path, ["*"]); + + // Should only include text file + expect(result.files.length).toBe(1); + expect(result.files[0]?.path).toBe("text.txt"); + + // Binary file should be in errors + expect(result.errors.length).toBe(1); + expect(result.errors[0]?.pattern).toBe("binary.dat"); + expect(result.errors[0]?.error).toContain("Binary"); + }); + + test("returns empty for non-matching patterns", async () => { + using tempDir = new DisposableTempDir("include-files-empty"); + + await fs.writeFile(path.join(tempDir.path, "file.txt"), "content"); + + const runtime = new LocalRuntime(tempDir.path); + const result = await resolveIncludeFiles(runtime, tempDir.path, ["*.nonexistent"]); + + expect(result.files).toHaveLength(0); + expect(result.errors).toHaveLength(0); // No match is not an error + }); + }); + + describe("renderIncludedFilesContext", () => { + test("renders all files and errors as XML", () => { + const result = renderIncludedFilesContext({ + files: [ + { path: "a.ts", content: "const a = 1;", truncated: false }, + { path: "b.json", content: '{"b": 2}', truncated: true }, + ], + errors: [{ pattern: "c.bin", error: "Binary file" }], + }); + + expect(result).toContain("<@a.ts>"); + expect(result).toContain("<@b.json"); + expect(result).toContain('truncated="true"'); + expect(result).toContain('<@c.bin error="Binary file" />'); + }); + }); +}); diff --git a/src/node/services/agentSkills/includeFilesResolver.ts b/src/node/services/agentSkills/includeFilesResolver.ts new file mode 100644 index 0000000000..bfae70c96d --- /dev/null +++ b/src/node/services/agentSkills/includeFilesResolver.ts @@ -0,0 +1,304 @@ +/** + * Resolves `include_files` glob patterns for agent skills into file content. + * + * Provides a unified XML representation for context files using the `<@path>` tag format. + */ +import * as path from "path"; +import picomatch from "picomatch"; + +import type { Runtime } from "@/node/runtime/Runtime"; +import { SSHRuntime } from "@/node/runtime/SSHRuntime"; +import { shellQuote } from "@/node/runtime/backgroundCommands"; +import { execBuffered, readFileString } from "@/node/utils/runtime/helpers"; +import { log } from "@/node/services/log"; + +// Conservative limits for included files +const MAX_INCLUDE_FILES = 20; +const MAX_BYTES_PER_FILE = 32 * 1024; // 32KB per file +const MAX_TOTAL_BYTES = 128 * 1024; // 128KB total across all included files +const MAX_LINES_PER_FILE = 500; + +export interface IncludedFile { + /** Path relative to skill directory */ + path: string; + /** File content */ + content: string; + /** Whether content was truncated */ + truncated: boolean; +} + +export interface IncludeFilesResult { + files: IncludedFile[]; + /** Patterns that had errors during expansion */ + errors: Array<{ pattern: string; error: string }>; +} + +/** + * Guess code fence language from file extension for syntax highlighting. + */ +function guessCodeFenceLanguage(filePath: string): string { + const ext = path.extname(filePath).toLowerCase(); + const langMap: Record = { + ".ts": "ts", + ".tsx": "tsx", + ".js": "js", + ".jsx": "jsx", + ".json": "json", + ".md": "md", + ".yml": "yaml", + ".yaml": "yaml", + ".sh": "sh", + ".bash": "bash", + ".py": "py", + ".go": "go", + ".rs": "rs", + ".css": "css", + ".html": "html", + ".xml": "xml", + ".sql": "sql", + ".rb": "rb", + ".java": "java", + ".c": "c", + ".cpp": "cpp", + ".h": "h", + ".hpp": "hpp", + }; + return langMap[ext] ?? ""; +} + +function encodeAtTagNamePath(filePath: string): string { + // Encode characters that can break tag parsing (whitespace, quotes, etc.). + // Keep common path separators readable. + return filePath.replace(/[^A-Za-z0-9._/-]/g, (character) => { + return Array.from(Buffer.from(character, "utf8")) + .map((byte) => `%${byte.toString(16).toUpperCase().padStart(2, "0")}`) + .join(""); + }); +} + +/** + * Render a file as unified context XML using the `<@path>` tag format. + */ +export function renderContextFile(file: IncludedFile): string { + const lang = guessCodeFenceLanguage(file.path); + const fence = lang ? `\`\`\`${lang}` : "```"; + const truncatedAttr = file.truncated ? ' truncated="true"' : ""; + const tagPath = encodeAtTagNamePath(file.path); + + return ( + `<@${tagPath}${truncatedAttr}>\n` + + `${fence}\n` + + `${file.content}\n` + + `\`\`\`\n` + + `` + ); +} + +/** + * Render an error for a file that couldn't be included. + */ +export function renderContextFileError(filePath: string, error: string): string { + const tagPath = encodeAtTagNamePath(filePath); + return `<@${tagPath} error="${escapeXmlAttr(error)}" />`; +} + +function escapeXmlAttr(value: string): string { + return value + .replace(/&/g, "&") + .replace(/"/g, """) + .replace(//g, ">"); +} + +/** + * List files in directory matching patterns using find command. + * Falls back to simple file listing if find fails. + */ +async function listFilesInDir(runtime: Runtime, dir: string, cwd: string): Promise { + const quotedDir = shellQuote(dir); + const command = + `if [ -d ${quotedDir} ]; then ` + + `find ${quotedDir} -type f -maxdepth 5 2>/dev/null | head -500; ` + + `fi`; + + const result = await execBuffered(runtime, command, { cwd, timeout: 10 }); + if (result.exitCode !== 0) { + log.warn(`Failed to list files in ${dir}: ${result.stderr || result.stdout}`); + return []; + } + + return result.stdout + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); +} + +async function listFilesWithGitLsFiles(runtime: Runtime, cwd: string): Promise { + const result = await execBuffered(runtime, "git ls-files -co --exclude-standard", { + cwd, + timeout: 10, + }); + if (result.exitCode !== 0) { + log.warn(`Failed to list files with git ls-files in ${cwd}: ${result.stderr || result.stdout}`); + return []; + } + + return result.stdout + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); +} + +/** + * Resolve glob patterns against a skill directory and return matching files. + */ +export async function resolveIncludeFiles( + runtime: Runtime, + skillDir: string, + patterns: string[], + options?: { abortSignal?: AbortSignal; listMode?: "find" | "git" } +): Promise { + if (!patterns || patterns.length === 0) { + return { files: [], errors: [] }; + } + + const resolvedSkillDir = await runtime.resolvePath(skillDir); + const pathModule = runtime instanceof SSHRuntime ? path.posix : path; + + const listMode = options?.listMode ?? "find"; + + const toRelativePaths = (files: string[]): string[] => { + return files + .map((f) => { + const rel = pathModule.relative(resolvedSkillDir, f); + // Normalize to forward slashes for consistent glob matching + return rel.replace(/\\/g, "/"); + }) + .filter((rel) => rel && !rel.startsWith("..")); + }; + + let relativePaths: string[]; + if (listMode === "git") { + const gitPaths = await listFilesWithGitLsFiles(runtime, resolvedSkillDir); + relativePaths = + gitPaths.length > 0 + ? gitPaths.map((filePath) => filePath.replace(/\\/g, "/")).filter(Boolean) + : toRelativePaths(await listFilesInDir(runtime, resolvedSkillDir, resolvedSkillDir)); + } else { + // List all files in the skill directory (up to reasonable depth) + relativePaths = toRelativePaths( + await listFilesInDir(runtime, resolvedSkillDir, resolvedSkillDir) + ); + } + + // Match patterns using picomatch + const matchedPaths = new Set(); + const errors: Array<{ pattern: string; error: string }> = []; + + for (const pattern of patterns) { + try { + const matcher = picomatch(pattern, { dot: true }); + let hasMatch = false; + + for (const relPath of relativePaths) { + if (matcher(relPath)) { + matchedPaths.add(relPath); + hasMatch = true; + } + } + + if (!hasMatch) { + // Not an error, just no matches - common for optional patterns + log.debug(`include_files pattern '${pattern}' matched no files in ${skillDir}`); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + errors.push({ pattern, error: `Invalid pattern: ${message}` }); + } + } + + // Read matched files respecting limits + const files: IncludedFile[] = []; + let totalBytes = 0; + + const sortedPaths = Array.from(matchedPaths).sort(); + + for (const relPath of sortedPaths) { + if (files.length >= MAX_INCLUDE_FILES) { + log.debug(`include_files: hit max file limit (${MAX_INCLUDE_FILES})`); + break; + } + + if (totalBytes >= MAX_TOTAL_BYTES) { + log.debug(`include_files: hit total bytes limit (${MAX_TOTAL_BYTES})`); + break; + } + + const fullPath = pathModule.join(resolvedSkillDir, relPath); + + try { + const stat = await runtime.stat(fullPath, options?.abortSignal); + if (stat.isDirectory) continue; + + if (stat.size > MAX_BYTES_PER_FILE) { + errors.push({ + pattern: relPath, + error: `File too large (${(stat.size / 1024).toFixed(1)}KB > ${MAX_BYTES_PER_FILE / 1024}KB)`, + }); + continue; + } + + let content = await readFileString(runtime, fullPath, options?.abortSignal); + + // Check for binary content + if (content.includes("\u0000")) { + errors.push({ pattern: relPath, error: "Binary file detected" }); + continue; + } + + // Apply line limits + let truncated = false; + const lines = content.split("\n"); + if (lines.length > MAX_LINES_PER_FILE) { + content = lines.slice(0, MAX_LINES_PER_FILE).join("\n"); + truncated = true; + } + + // Check total bytes budget + const contentBytes = Buffer.byteLength(content, "utf8"); + if (totalBytes + contentBytes > MAX_TOTAL_BYTES) { + // Truncate content to fit remaining budget + const remaining = MAX_TOTAL_BYTES - totalBytes; + if (remaining < 100) break; // Not worth truncating to tiny amount + + content = Buffer.from(content, "utf8").subarray(0, remaining).toString("utf8"); + truncated = true; + } + + files.push({ path: relPath, content, truncated }); + totalBytes += Buffer.byteLength(content, "utf8"); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + errors.push({ pattern: relPath, error: message }); + } + } + + return { files, errors }; +} + +/** + * Render all included files as unified context XML. + */ +export function renderIncludedFilesContext(result: IncludeFilesResult): string { + const blocks: string[] = []; + + for (const file of result.files) { + blocks.push(renderContextFile(file)); + } + + for (const error of result.errors) { + blocks.push(renderContextFileError(error.pattern, error.error)); + } + + return blocks.join("\n\n"); +} diff --git a/src/node/services/agentSkills/parseSkillMarkdown.test.ts b/src/node/services/agentSkills/parseSkillMarkdown.test.ts index f7a5310e09..e11d801ae5 100644 --- a/src/node/services/agentSkills/parseSkillMarkdown.test.ts +++ b/src/node/services/agentSkills/parseSkillMarkdown.test.ts @@ -75,4 +75,68 @@ Body }) ).toThrow(AgentSkillParseError); }); + + test("parses include_files patterns", () => { + const content = `--- +name: with-files +description: Skill with included files +include_files: + - "examples/*.ts" + - "schemas/**/*.json" +--- +Body +`; + + const directoryName = SkillNameSchema.parse("with-files"); + + const result = parseSkillMarkdown({ + content, + byteSize: Buffer.byteLength(content, "utf-8"), + directoryName, + }); + + expect(result.frontmatter.include_files).toEqual(["examples/*.ts", "schemas/**/*.json"]); + }); + + test("rejects absolute paths in include_files", () => { + const content = `--- +name: bad-paths +description: Should fail +include_files: + - "/etc/passwd" +--- +Body +`; + + const directoryName = SkillNameSchema.parse("bad-paths"); + + expect(() => + parseSkillMarkdown({ + content, + byteSize: Buffer.byteLength(content, "utf-8"), + directoryName, + }) + ).toThrow(AgentSkillParseError); + }); + + test("rejects path traversal in include_files", () => { + const content = `--- +name: traversal +description: Should fail +include_files: + - "../secret.txt" +--- +Body +`; + + const directoryName = SkillNameSchema.parse("traversal"); + + expect(() => + parseSkillMarkdown({ + content, + byteSize: Buffer.byteLength(content, "utf-8"), + directoryName, + }) + ).toThrow(AgentSkillParseError); + }); }); diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index f31223f369..9b9201de2c 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -30,6 +30,7 @@ import type { MuxProviderOptions } from "@/common/types/providerOptions"; import type { BackgroundProcessManager } from "@/node/services/backgroundProcessManager"; import type { FileState, EditedFileAttachment } from "@/node/services/agentSession"; import { log } from "./log"; +import { injectAgentIncludeFiles } from "./agentIncludeFiles"; import { injectFileAtMentions } from "./fileAtMentions"; import { transformModelMessages, @@ -80,6 +81,7 @@ import { readPlanFile } from "@/node/utils/runtime/helpers"; import { readAgentDefinition, resolveAgentBody, + resolveAgentIncludeFilesPatterns, } from "@/node/services/agentDefinitions/agentDefinitionsService"; import { resolveToolPolicyForAgent } from "@/node/services/agentDefinitions/resolveToolPolicy"; import { isPlanLike } from "@/common/utils/agentInheritance"; @@ -1296,9 +1298,34 @@ export class AIService extends EventEmitter { postCompactionAttachments ); + // Inject agent include_files context as an in-memory synthetic user message. + // This keeps the system message stable (cache-friendly), while still giving the model + // immediate, structured file context. + let messagesWithAgentIncludeFiles = messagesWithPostCompaction; + try { + const includeFilePatterns = await resolveAgentIncludeFilesPatterns( + runtime, + agentDiscoveryPath, + agentDefinition.id + ); + + messagesWithAgentIncludeFiles = await injectAgentIncludeFiles(messagesWithPostCompaction, { + runtime, + workspacePath, + patterns: includeFilePatterns, + abortSignal, + }); + } catch (error) { + log.warn("Failed to inject agent include_files; continuing without it", { + workspaceId, + agentId: agentDefinition.id, + error: error instanceof Error ? error.message : String(error), + }); + } + // Expand @file mentions (e.g. @src/foo.ts#L1-20) into an in-memory synthetic user message. // This keeps chat history clean while giving the model immediate file context. - const messagesWithFileAtMentions = await injectFileAtMentions(messagesWithPostCompaction, { + const messagesWithFileAtMentions = await injectFileAtMentions(messagesWithAgentIncludeFiles, { runtime, workspacePath, abortSignal, diff --git a/src/node/services/fileAtMentions.test.ts b/src/node/services/fileAtMentions.test.ts index 3c7ea7ba04..f484acdc7a 100644 --- a/src/node/services/fileAtMentions.test.ts +++ b/src/node/services/fileAtMentions.test.ts @@ -34,7 +34,7 @@ describe("injectFileAtMentions", () => { expect(result[1]).toEqual(messages[0]); const injectedText = result[0]?.parts.find((p) => p.type === "text")?.text ?? ""; - expect(injectedText).toContain(' { expect(result[0]?.metadata?.synthetic).toBe(true); const injectedText = result[0]?.parts.find((p) => p.type === "text")?.text ?? ""; - expect(injectedText).toContain('/g, ">"); +} + +function encodeAtTagNamePath(filePath: string): string { + // Encode characters that can break tag parsing (whitespace, quotes, etc.). + // Keep common path separators readable. + return filePath.replace(/[^A-Za-z0-9._/-]/g, (character) => { + return Array.from(Buffer.from(character, "utf8")) + .map((byte) => `%${byte.toString(16).toUpperCase().padStart(2, "0")}`) + .join(""); + }); +} + +function renderAtTaggedFileBlock(options: { filePath: string; rangeLabel: string; content: string; @@ -149,18 +167,21 @@ function renderMuxFileBlock(options: { const lang = guessCodeFenceLanguage(options.filePath); const fence = lang ? `\`\`\`${lang}` : "```"; const truncatedAttr = options.truncated ? ' truncated="true"' : ""; + const rangeAttr = options.rangeLabel ? ` range="${escapeXmlAttr(options.rangeLabel)}"` : ""; + const tagPath = encodeAtTagNamePath(options.filePath); return ( - `\n` + + `<@${tagPath}${rangeAttr}${truncatedAttr}>\n` + `${fence}\n` + `${options.content}\n` + `\`\`\`\n` + - `` + `` ); } -function renderMuxFileError(filePath: string, error: string): string { - return `${error}`; +function renderAtTaggedFileError(filePath: string, error: string): string { + const tagPath = encodeAtTagNamePath(filePath); + return `<@${tagPath} error="${escapeXmlAttr(error)}" />`; } export async function injectFileAtMentions( @@ -252,7 +273,7 @@ export async function injectFileAtMentions( continue; } - const block = renderMuxFileError(displayPath, mention.rangeError); + const block = renderAtTaggedFileError(displayPath, mention.rangeError); const blockBytes = Buffer.byteLength(block, "utf8"); if (totalBytes + blockBytes > MAX_TOTAL_BYTES) { break; @@ -267,7 +288,7 @@ export async function injectFileAtMentions( resolvedPath = resolveWorkspaceFilePath(options.runtime, options.workspacePath, mention.path); } catch (error) { const message = error instanceof Error ? error.message : String(error); - const block = renderMuxFileError(displayPath, message); + const block = renderAtTaggedFileError(displayPath, message); const blockBytes = Buffer.byteLength(block, "utf8"); if (totalBytes + blockBytes > MAX_TOTAL_BYTES) { break; @@ -286,7 +307,7 @@ export async function injectFileAtMentions( } const message = error instanceof Error ? error.message : String(error); - const block = renderMuxFileError(displayPath, `Failed to stat file: ${message}`); + const block = renderAtTaggedFileError(displayPath, `Failed to stat file: ${message}`); const blockBytes = Buffer.byteLength(block, "utf8"); if (totalBytes + blockBytes > MAX_TOTAL_BYTES) { break; @@ -301,7 +322,7 @@ export async function injectFileAtMentions( continue; } - const block = renderMuxFileError(displayPath, "Path is a directory, not a file."); + const block = renderAtTaggedFileError(displayPath, "Path is a directory, not a file."); const blockBytes = Buffer.byteLength(block, "utf8"); if (totalBytes + blockBytes > MAX_TOTAL_BYTES) { break; @@ -314,7 +335,7 @@ export async function injectFileAtMentions( if (stat.size > MAX_FILE_SIZE) { const sizeMB = (stat.size / (1024 * 1024)).toFixed(2); const maxMB = (MAX_FILE_SIZE / (1024 * 1024)).toFixed(2); - const block = renderMuxFileError( + const block = renderAtTaggedFileError( displayPath, `File is too large to include (${sizeMB}MB > ${maxMB}MB). Use a smaller #L- range or file_read.` ); @@ -332,7 +353,7 @@ export async function injectFileAtMentions( content = await readFileString(options.runtime, resolvedPath, options.abortSignal); } catch (error) { const message = error instanceof Error ? error.message : String(error); - const block = renderMuxFileError(displayPath, `Failed to read file: ${message}`); + const block = renderAtTaggedFileError(displayPath, `Failed to read file: ${message}`); const blockBytes = Buffer.byteLength(block, "utf8"); if (totalBytes + blockBytes > MAX_TOTAL_BYTES) { break; @@ -343,7 +364,10 @@ export async function injectFileAtMentions( } if (content.includes("\u0000")) { - const block = renderMuxFileError(displayPath, "Binary file detected (NUL byte). Skipping."); + const block = renderAtTaggedFileError( + displayPath, + "Binary file detected (NUL byte). Skipping." + ); const blockBytes = Buffer.byteLength(block, "utf8"); if (totalBytes + blockBytes > MAX_TOTAL_BYTES) { break; @@ -360,7 +384,7 @@ export async function injectFileAtMentions( const requestedEnd = mention.range?.endLine ?? Math.max(1, lines.length); if (lines.length > 0 && requestedStart > lines.length) { - const block = renderMuxFileError( + const block = renderAtTaggedFileError( displayPath, `Range starts beyond end of file: requested L${requestedStart}, file has ${lines.length} lines.` ); @@ -402,7 +426,7 @@ export async function injectFileAtMentions( const rangeStart = requestedStart; const rangeEnd = processedLines.length > 0 ? requestedStart + processedLines.length - 1 : 0; const rangeLabel = formatRange(rangeStart, rangeEnd, processedLines.length); - const header = renderMuxFileBlock({ + const header = renderAtTaggedFileBlock({ filePath: displayPath, rangeLabel, content: "", @@ -423,7 +447,7 @@ export async function injectFileAtMentions( const finalRangeEnd = finalLines.length > 0 ? requestedStart + finalLines.length - 1 : 0; const finalRangeLabel = formatRange(requestedStart, finalRangeEnd, finalLines.length); - const block = renderMuxFileBlock({ + const block = renderAtTaggedFileBlock({ filePath: displayPath, rangeLabel: finalRangeLabel, content: finalLines.join("\n"),