De-dupe skill utils (#1150)

Author: JasonYeMSFT

scripts/src/copilot-cli-char-budget.ts

@@ -3,8 +3,8 @@
  * See https://github.com/github/copilot-agent-runtime/blob/0e43d66f7570421ba4b27a36a86ea908de188e59/src/skills/skillToolDescription.ts#L30
  */
 
-import { listSkills, loadSkill } from "./helpers/skill-helper.js";
-import { escapeXml } from "./helpers/string-helpers.js";
+import { listSkills, loadSkill } from "./shared/skill-helper.js";
+import { escapeXml } from "./shared/string-helpers.js";
 
 const SKILL_CHAR_BUDGET_ENV = "SKILL_CHAR_BUDGET";
 const DEFAULT_SKILL_CHAR_BUDGET = 15000;

scripts/src/frontmatter/__tests__/frontmatter.test.ts

@@ -14,7 +14,7 @@ import {
   validateMetadataVersion,
   validateSkillFile,
 } from "../cli.js";
-import { parseSkillContent } from "../../shared/parse-skill.js";
+import { parseSkillContent } from "../../shared/skill-helper.js";
 
 const TEST_DIR = resolve(__dirname, "__test_frontmatter__");
 

scripts/src/frontmatter/cli.ts

@@ -20,7 +20,7 @@
 import { dirname, resolve, basename } from "node:path";
 import { fileURLToPath } from "node:url";
 import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
-import { parseSkillContent } from "../shared/parse-skill.js";
+import { parseSkillContent } from "../shared/skill-helper.js";
 
 // ── Paths ────────────────────────────────────────────────────────────────────
 

scripts/src/helpers/skill-helper.ts

@@ -11,7 +11,7 @@ import { existsSync, readdirSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { homedir } from "node:os";
 import { execFileSync } from "node:child_process";
-import { parseSkillContent } from "../../shared/parse-skill.js";
+import { parseSkillContent } from "../../shared/skill-helper.js";
 
 const TIMEOUT_MS = 120_000;
 const MARKETPLACE_NAME = "github-copilot-for-azure";
@@ -62,7 +62,7 @@ function checkPluginConfig(expectedCachePath: string): TestResult {
 
   const marketplace = config.marketplaces?.[MARKETPLACE_NAME];
   if (!marketplace || marketplace.source?.source !== "github" ||
-      marketplace.source?.repo !== "microsoft/github-copilot-for-azure") {
+    marketplace.source?.repo !== "microsoft/github-copilot-for-azure") {
     return { name: "Plugin config", passed: false, detail: `Marketplace "${MARKETPLACE_NAME}" not configured correctly` };
   }
 

scripts/src/local/commands/test.ts

@@ -18,7 +18,7 @@ import {
 import { join } from "node:path";
 import { homedir } from "node:os";
 import { setup } from "./setup.js";
-import { parseSkillContent } from "../../shared/parse-skill.js";
+import { parseSkillContent } from "../../shared/skill-helper.js";
 
 const MARKETPLACE_NAME = "github-copilot-for-azure";
 const PLUGIN_NAME = "azure";

scripts/src/local/commands/verify.ts

@@ -3,7 +3,7 @@
  */
 
 import { describe, it, expect } from "vitest";
-import { parseSkillContent } from "../parse-skill.js";
+import { parseSkillContent } from "../skill-helper.js";
 
 describe("parseSkillContent", () => {
   it("parses valid frontmatter", () => {

scripts/src/shared/__tests__/parse-skill.test.ts

@@ -0,0 +1,144 @@
+/**
+ * Skill Utility
+ *
+ * Shared helpers for loading, listing, and parsing SKILL.md files from the
+ * plugin/skills directory.  All frontmatter parsing goes through
+ * `parseSkillContent` which normalises line endings, validates `---`
+ * delimiters, and exposes the raw YAML source.
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { fileURLToPath } from "node:url";
+import matter from "gray-matter";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// ── Types ────────────────────────────────────────────────────────────────────
+
+/** Parsed frontmatter result from a SKILL.md file. */
+export interface ParsedSkill {
+  /** Parsed key-value pairs (name, description, …). */
+  data: Record<string, unknown>;
+  /** Markdown body after the closing `---`. */
+  content: string;
+  /**
+   * Raw frontmatter text between the `---` delimiters (not including the
+   * delimiters themselves).  Useful for checks that need the original YAML
+   * source — e.g. detecting block scalars (`>-`, `|`) or XML-like tags.
+   */
+  raw: string;
+}
+
+interface SkillMetadata {
+  name: string;
+  description: string;
+  [key: string]: unknown;
+}
+
+export interface LoadedSkill {
+  metadata: SkillMetadata;
+  content: string;
+  path: string;
+  filePath: string;
+}
+
+// ── Parser ───────────────────────────────────────────────────────────────────
+
+/**
+ * Parse SKILL.md file content and extract frontmatter.
+ *
+ * - Normalises `\r\n` → `\n` before parsing.
+ * - Returns `null` when the file does not contain valid `---` delimited
+ *   frontmatter (instead of throwing).
+ */
+export function parseSkillContent(fileContent: string): ParsedSkill | null {
+  // Normalise Windows line-endings
+  const normalised = fileContent.replace(/\r\n/g, "\n");
+
+  // Quick guard: gray-matter is lenient — we require the file to start
+  // with `---` (the agentskills.io spec mandates it).
+  if (!normalised.startsWith("---")) return null;
+
+  // Also require a closing `---` delimiter.  gray-matter treats
+  // EOF as an implicit close, but the spec requires explicit delimiters.
+  const closingIndex = normalised.indexOf("\n---", 3);
+  if (closingIndex === -1) return null;
+
+  try {
+    const result = matter(normalised);
+
+    // gray-matter sets `data` to `{}` when there is no frontmatter or
+    // when the delimiters are malformed.  Treat that as "no frontmatter".
+    if (Object.keys(result.data).length === 0) return null;
+
+    // Extract the raw YAML block.  gray-matter exposes `result.matter` in
+    // recent versions but its behaviour across versions is inconsistent,
+    // so we derive it ourselves from the normalised input.
+    const raw = normalised.substring(4, closingIndex);
+
+    return {
+      data: result.data as Record<string, unknown>,
+      content: result.content,
+      raw,
+    };
+  } catch {
+    // YAML parse error → treat as "no valid frontmatter"
+    return null;
+  }
+}
+
+// ── Loaders ──────────────────────────────────────────────────────────────────
+
+/**
+ * Load a skill by name.
+ *
+ * Reads the SKILL.md file from `plugin/skills/<skillName>` and parses it
+ * via `parseSkillContent`.  Throws when the file is missing or contains
+ * no valid frontmatter.
+ */
+export function loadSkill(skillName: string): LoadedSkill {
+  const skillPath = path.join(
+    path.resolve(__dirname, "../../../plugin/skills"),
+    skillName
+  );
+  const skillFile = path.join(skillPath, "SKILL.md");
+
+  if (!fs.existsSync(skillFile)) {
+    throw new Error(`SKILL.md not found for skill: ${skillName} at ${skillFile}`);
+  }
+
+  const fileContent = fs.readFileSync(skillFile, "utf-8");
+  const parsed = parseSkillContent(fileContent);
+
+  if (!parsed) {
+    throw new Error(`Invalid or missing frontmatter in SKILL.md for skill: ${skillName}`);
+  }
+
+  return {
+    metadata: {
+      name: (parsed.data.name as string) || skillName,
+      description: (parsed.data.description as string) || "",
+      ...parsed.data
+    },
+    content: parsed.content.trim(),
+    path: skillPath,
+    filePath: skillFile
+  };
+}
+
+/**
+ * @returns Names of skills in azure plugin.
+ */
+export function listSkills(): string[] {
+  const skillsDir = path.resolve(__dirname, "../../../plugin/skills");
+  const items = fs.readdirSync(skillsDir, { withFileTypes: true });
+  return items
+    .filter((item) => item.isDirectory())
+    .filter((item) => {
+      const skillMdPath = path.join(skillsDir, item.name, "SKILL.md");
+      return fs.existsSync(skillMdPath);
+    })
+    .map((item) => item.name);
+}

scripts/src/shared/skill-helper.ts

@@ -45,7 +45,7 @@ describe("loadConfig", () => {
       }
     };
     writeFileSync(join(TEST_DIR, ".token-limits.json"), JSON.stringify(testConfig));
-    
+
     const config = loadConfig(TEST_DIR);
     expect(config.defaults["*.md"]).toBe(1500);
     expect(config.defaults["SKILL.md"]).toBe(3000);
@@ -54,15 +54,15 @@ describe("loadConfig", () => {
 
   it("returns defaults for invalid JSON", () => {
     writeFileSync(join(TEST_DIR, ".token-limits.json"), "not valid json");
-    
+
     const config = loadConfig(TEST_DIR);
     expect(config.defaults).toBeDefined();
     expect(config.defaults["*.md"]).toBeDefined();
   });
 
   it("returns defaults for config missing defaults field", () => {
     writeFileSync(join(TEST_DIR, ".token-limits.json"), JSON.stringify({ overrides: {} }));
-    
+
     const config = loadConfig(TEST_DIR);
     expect(config.defaults).toBeDefined();
   });
@@ -118,7 +118,7 @@ describe("getLimitForFile", () => {
       },
       overrides: {}
     };
-    
+
     const result = getLimitForFile("plugin/skills/my-skill/SKILL.md", configWithMultiplePatterns, "/root");
     // SKILL.md exact filename match wins over globstar pattern due to specificity scoring
     // (no wildcards = +10000 points)
@@ -144,7 +144,7 @@ describe("findMarkdownFiles", () => {
     writeFileSync(join(TEST_DIR, "README.md"), "# Test");
     writeFileSync(join(TEST_DIR, "GUIDE.md"), "# Guide");
     writeFileSync(join(TEST_DIR, "script.ts"), 'console.log("hi")');
-    
+
     const files = findMarkdownFiles(TEST_DIR);
     expect(files.length).toBe(2);
     expect(files.some(f => f.endsWith("README.md"))).toBe(true);
@@ -153,7 +153,7 @@ describe("findMarkdownFiles", () => {
 
   it("finds .mdx files", () => {
     writeFileSync(join(TEST_DIR, "component.mdx"), "# MDX");
-    
+
     const files = findMarkdownFiles(TEST_DIR);
     expect(files.length).toBe(1);
     expect(files[0].endsWith("component.mdx")).toBe(true);
@@ -164,7 +164,7 @@ describe("findMarkdownFiles", () => {
     writeFileSync(join(TEST_DIR, "root.md"), "# Root");
     writeFileSync(join(TEST_DIR, "sub", "sub.md"), "# Sub");
     writeFileSync(join(TEST_DIR, "sub", "deep", "deep.md"), "# Deep");
-    
+
     const files = findMarkdownFiles(TEST_DIR);
     expect(files.length).toBe(3);
   });
@@ -173,7 +173,7 @@ describe("findMarkdownFiles", () => {
     mkdirSync(join(TEST_DIR, "node_modules"));
     writeFileSync(join(TEST_DIR, "README.md"), "# Test");
     writeFileSync(join(TEST_DIR, "node_modules", "pkg.md"), "# Package");
-    
+
     const files = findMarkdownFiles(TEST_DIR);
     expect(files.length).toBe(1);
     expect(files[0].endsWith("README.md")).toBe(true);
@@ -183,7 +183,7 @@ describe("findMarkdownFiles", () => {
     mkdirSync(join(TEST_DIR, ".git"));
     writeFileSync(join(TEST_DIR, "README.md"), "# Test");
     writeFileSync(join(TEST_DIR, ".git", "config.md"), "# Config");
-    
+
     const files = findMarkdownFiles(TEST_DIR);
     expect(files.length).toBe(1);
   });
@@ -196,7 +196,7 @@ describe("findMarkdownFiles", () => {
   it("returns empty array for directory with no markdown files", () => {
     writeFileSync(join(TEST_DIR, "script.ts"), "code");
     writeFileSync(join(TEST_DIR, "data.json"), "{}");
-    
+
     const files = findMarkdownFiles(TEST_DIR);
     expect(files).toEqual([]);
   });