Skip to content

Commit dba8f95

Browse files
Fix skill loader GitHub fallback branch and add tests
The factory-plugins repo uses 'master' as its default branch, not 'main'. The GitHub fallback was always 404ing in CI. Added unit tests for the loader functions and an E2E test that verifies the skill downloads from GitHub with no local cache (simulating a CI runner). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
1 parent 7c007f8 commit dba8f95

File tree

3 files changed

+264
-1
lines changed

3 files changed

+264
-1
lines changed

src/utils/load-skill.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { homedir } from "os";
33
import { resolve, join } from "path";
44

55
const PLUGINS_REPO = "Factory-AI/factory-plugins";
6-
const PLUGINS_BRANCH = "main";
6+
const PLUGINS_BRANCH = "master";
77

88
const SHARED_BEGIN = "<!-- BEGIN_SHARED_METHODOLOGY -->";
99
const SHARED_END = "<!-- END_SHARED_METHODOLOGY -->";
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import {
2+
afterEach,
3+
beforeEach,
4+
describe,
5+
expect,
6+
it,
7+
} from "bun:test";
8+
import { mkdtemp, rm } from "fs/promises";
9+
import { join } from "path";
10+
import os from "os";
11+
import {
12+
loadSkill,
13+
formatSkillSection,
14+
extractSharedMethodology,
15+
} from "../../src/utils/load-skill";
16+
17+
/**
18+
* E2E test that simulates what the GitHub Action does:
19+
* - No local plugin cache (fresh CI runner)
20+
* - loadSkill("review") must succeed via GitHub fallback
21+
* - formatSkillSection must produce usable prompt content
22+
*
23+
* This mirrors the code path in:
24+
* src/entrypoints/generate-review-prompt.ts
25+
* src/tag/commands/review.ts
26+
* src/tag/commands/review-validator.ts
27+
*/
28+
describe("skill loading E2E (GitHub Action simulation)", () => {
29+
let fakeHome: string;
30+
const originalHome = process.env.HOME;
31+
32+
beforeEach(async () => {
33+
// Use a temp dir as HOME so there's no local plugin cache,
34+
// forcing the GitHub fallback path (same as a CI runner)
35+
fakeHome = await mkdtemp(join(os.tmpdir(), "skill-e2e-"));
36+
process.env.HOME = fakeHome;
37+
});
38+
39+
afterEach(async () => {
40+
if (originalHome !== undefined) {
41+
process.env.HOME = originalHome;
42+
} else {
43+
delete process.env.HOME;
44+
}
45+
await rm(fakeHome, { recursive: true, force: true });
46+
});
47+
48+
it("loads the review skill from GitHub when no local cache exists", async () => {
49+
const content = await loadSkill("review");
50+
51+
expect(content).toBeDefined();
52+
expect(content.length).toBeGreaterThan(500);
53+
});
54+
55+
it("extracts shared methodology with expected markers", async () => {
56+
const content = await loadSkill("review");
57+
const methodology = extractSharedMethodology(content);
58+
59+
// These sections must be present for the review prompts to work
60+
expect(methodology).toContain("Reporting Gate");
61+
expect(methodology).toContain("Bug Patterns");
62+
expect(methodology).toContain("Confidence calibration");
63+
expect(methodology).toContain("P0");
64+
expect(methodology).toContain("P1");
65+
expect(methodology).toContain("P2");
66+
});
67+
68+
it("formatSkillSection produces non-empty prompt section", async () => {
69+
const content = await loadSkill("review");
70+
const section = formatSkillSection(content);
71+
72+
expect(section).toContain("<code_review_methodology>");
73+
expect(section).toContain("</code_review_methodology>");
74+
// The methodology must be substantial enough to replace the
75+
// previously inlined guidance that was removed from the templates
76+
expect(section.length).toBeGreaterThan(200);
77+
});
78+
79+
it("methodology does NOT include the full skill (only shared section)", async () => {
80+
const content = await loadSkill("review");
81+
const methodology = extractSharedMethodology(content);
82+
83+
// The shared methodology should not include the YAML frontmatter
84+
// or the Two-Pass Review Pipeline section (those are skill-only)
85+
expect(methodology).not.toContain("name: review");
86+
expect(methodology).not.toContain("version:");
87+
});
88+
});

test/utils/load-skill.test.ts

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
import {
2+
afterEach,
3+
beforeEach,
4+
describe,
5+
expect,
6+
it,
7+
spyOn,
8+
} from "bun:test";
9+
import * as fsPromises from "fs/promises";
10+
import {
11+
extractSharedMethodology,
12+
formatSkillSection,
13+
loadSkill,
14+
} from "../../src/utils/load-skill";
15+
16+
describe("extractSharedMethodology", () => {
17+
it("extracts content between markers", () => {
18+
const content = `
19+
# Skill Header
20+
21+
Some intro text.
22+
23+
<!-- BEGIN_SHARED_METHODOLOGY -->
24+
## Bug Patterns
25+
- Pattern 1
26+
- Pattern 2
27+
28+
## Reporting Gate
29+
Only report real bugs.
30+
<!-- END_SHARED_METHODOLOGY -->
31+
32+
## Other Section
33+
This should not be included.
34+
`;
35+
const result = extractSharedMethodology(content);
36+
expect(result).toContain("## Bug Patterns");
37+
expect(result).toContain("## Reporting Gate");
38+
expect(result).not.toContain("# Skill Header");
39+
expect(result).not.toContain("## Other Section");
40+
});
41+
42+
it("returns full content when markers are missing", () => {
43+
const content = "No markers here, just plain content.";
44+
const result = extractSharedMethodology(content);
45+
expect(result).toBe(content);
46+
});
47+
48+
it("returns full content when only BEGIN marker exists", () => {
49+
const content = "<!-- BEGIN_SHARED_METHODOLOGY -->\nSome content";
50+
const result = extractSharedMethodology(content);
51+
expect(result).toBe(content);
52+
});
53+
54+
it("returns full content when END comes before BEGIN", () => {
55+
const content =
56+
"<!-- END_SHARED_METHODOLOGY -->\nMiddle\n<!-- BEGIN_SHARED_METHODOLOGY -->";
57+
const result = extractSharedMethodology(content);
58+
expect(result).toBe(content);
59+
});
60+
});
61+
62+
describe("formatSkillSection", () => {
63+
it("returns empty string for undefined input", () => {
64+
expect(formatSkillSection(undefined)).toBe("");
65+
});
66+
67+
it("returns empty string for empty string input", () => {
68+
expect(formatSkillSection("")).toBe("");
69+
});
70+
71+
it("wraps extracted methodology in XML tags", () => {
72+
const content = `
73+
<!-- BEGIN_SHARED_METHODOLOGY -->
74+
## Reporting Gate
75+
Only report real bugs.
76+
<!-- END_SHARED_METHODOLOGY -->
77+
`;
78+
const result = formatSkillSection(content);
79+
expect(result).toContain("<code_review_methodology>");
80+
expect(result).toContain("</code_review_methodology>");
81+
expect(result).toContain("## Reporting Gate");
82+
});
83+
});
84+
85+
describe("loadSkill", () => {
86+
let readdirSpy: ReturnType<typeof spyOn>;
87+
let readFileSpy: ReturnType<typeof spyOn>;
88+
let fetchSpy: ReturnType<typeof spyOn>;
89+
const originalHome = process.env.HOME;
90+
91+
beforeEach(() => {
92+
process.env.HOME = "/mock-home";
93+
readdirSpy = spyOn(fsPromises, "readdir");
94+
readFileSpy = spyOn(fsPromises, "readFile");
95+
fetchSpy = spyOn(globalThis, "fetch");
96+
});
97+
98+
afterEach(() => {
99+
readdirSpy.mockRestore();
100+
readFileSpy.mockRestore();
101+
fetchSpy.mockRestore();
102+
if (originalHome !== undefined) {
103+
process.env.HOME = originalHome;
104+
} else {
105+
delete process.env.HOME;
106+
}
107+
});
108+
109+
it("loads skill from local cache when available", async () => {
110+
readdirSpy.mockResolvedValue(["abc123"] as any);
111+
readFileSpy.mockResolvedValue("cached skill content");
112+
113+
const result = await loadSkill("review");
114+
115+
expect(result).toBe("cached skill content");
116+
expect(fetchSpy).not.toHaveBeenCalled();
117+
});
118+
119+
it("falls back to GitHub when local cache is empty", async () => {
120+
readdirSpy.mockRejectedValue(new Error("ENOENT"));
121+
fetchSpy.mockResolvedValue({
122+
ok: true,
123+
text: () => Promise.resolve("github skill content"),
124+
} as Response);
125+
126+
const result = await loadSkill("review");
127+
128+
expect(result).toBe("github skill content");
129+
const fetchUrl = fetchSpy.mock.calls[0]![0] as string;
130+
expect(fetchUrl).toContain("/master/");
131+
expect(fetchUrl).not.toContain("/main/");
132+
});
133+
134+
it("throws when both sources fail", async () => {
135+
readdirSpy.mockRejectedValue(new Error("ENOENT"));
136+
fetchSpy.mockResolvedValue({ ok: false, status: 404 } as Response);
137+
138+
await expect(loadSkill("review")).rejects.toThrow(
139+
'Required skill "review" not found',
140+
);
141+
});
142+
143+
it("skips empty cache entries and tries next hash", async () => {
144+
readdirSpy.mockResolvedValue(["hash1", "hash2"] as any);
145+
readFileSpy
146+
.mockRejectedValueOnce(new Error("ENOENT"))
147+
.mockResolvedValueOnce("skill from hash2");
148+
149+
const result = await loadSkill("review");
150+
expect(result).toBe("skill from hash2");
151+
});
152+
});
153+
154+
describe("loadSkill GitHub integration", () => {
155+
let readdirSpy: ReturnType<typeof spyOn>;
156+
157+
beforeEach(() => {
158+
readdirSpy = spyOn(fsPromises, "readdir").mockRejectedValue(
159+
new Error("ENOENT"),
160+
);
161+
});
162+
163+
afterEach(() => {
164+
readdirSpy.mockRestore();
165+
});
166+
167+
it("fetches the review skill from the factory-plugins repo", async () => {
168+
const content = await loadSkill("review");
169+
170+
expect(content.length).toBeGreaterThan(100);
171+
expect(content).toContain("BEGIN_SHARED_METHODOLOGY");
172+
expect(content).toContain("END_SHARED_METHODOLOGY");
173+
expect(content).toContain("Reporting Gate");
174+
});
175+
});

0 commit comments

Comments
 (0)