Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 21 additions & 11 deletions cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -560,10 +560,10 @@ export async function runImportMarkdown(
try {
const stats = await fsPromises.stat(memoryDir);
if (stats.isDirectory()) {
const files = await fsPromises.readdir(memoryDir);
const files = await fsPromises.readdir(memoryDir, { withFileTypes: true });
for (const f of files) {
if (f.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f)) {
mdFiles.push({ filePath: path.join(memoryDir, f), scope: entry.name });
if (f.isFile() && f.name.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f.name)) {
mdFiles.push({ filePath: path.join(memoryDir, f.name), scope: entry.name });
}
}
}
Expand Down Expand Up @@ -592,10 +592,10 @@ export async function runImportMarkdown(
try {
const stats = await fsP.stat(agentMemoryDir);
if (stats.isDirectory()) {
const files = await fsP.readdir(agentMemoryDir);
const files = await fsP.readdir(agentMemoryDir, { withFileTypes: true });
for (const f of files) {
if (f.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f)) {
mdFiles.push({ filePath: path.join(agentMemoryDir, f), scope: agentId });
if (f.isFile() && f.name.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f.name)) {
mdFiles.push({ filePath: path.join(agentMemoryDir, f.name), scope: agentId });
}
}
}
Expand Down Expand Up @@ -633,10 +633,10 @@ export async function runImportMarkdown(
try {
const stats = await fsPromises.stat(flatMemoryDir);
if (stats.isDirectory()) {
const files = await fsPromises.readdir(flatMemoryDir);
const files = await fsPromises.readdir(flatMemoryDir, { withFileTypes: true });
for (const f of files) {
if (f.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f)) {
mdFiles.push({ filePath: path.join(flatMemoryDir, f), scope: workspaceScope || "global" });
if (f.isFile() && f.name.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f.name)) {
mdFiles.push({ filePath: path.join(flatMemoryDir, f.name), scope: workspaceScope || "global" });
}
}
}
Expand All @@ -658,8 +658,18 @@ export async function runImportMarkdown(

// Parse each file for memory entries (lines starting with "- ")
for (const { filePath, scope: discoveredScope } of mdFiles) {
foundFiles++;
let content = await fsPromises.readFile(filePath, "utf-8");
let content: string;
try {
// 已在收集時用 withFileTypes: true 過濾,直接讀取
foundFiles++;
content = await fsPromises.readFile(filePath, "utf-8");
} catch (err) {
// I/O errors (permissions, corruption, etc.)
console.warn(` [skip] read failed: ${filePath}: ${(err as Error).message}`);
skipped++;
continue;
}
// (fix(import-markdown): CI測試登記 + .md目錄skip保護)
// Strip UTF-8 BOM (e.g. from Windows Notepad-saved files)
content = content.replace(/^\uFEFF/, "");
// Normalize line endings: handle both CRLF (\r\n) and LF (\n)
Expand Down
1 change: 1 addition & 0 deletions scripts/ci-test-manifest.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ export const CI_TEST_MANIFEST = [
{ group: "core-regression", runner: "node", file: "test/recall-text-cleanup.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/update-consistency-lancedb.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/strip-envelope-metadata.test.mjs", args: ["--test"] },
{ group: "cli-smoke", runner: "node", file: "test/import-markdown/import-markdown.test.mjs", args: ["--test"] },
{ group: "cli-smoke", runner: "node", file: "test/cli-smoke.mjs" },
{ group: "cli-smoke", runner: "node", file: "test/functional-e2e.mjs" },
{ group: "core-regression", runner: "node", file: "test/retriever-rerank-regression.mjs" },
Expand Down
72 changes: 71 additions & 1 deletion test/import-markdown/import-markdown.test.mjs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/**
/**
* import-markdown.test.mjs
* Integration tests for the import-markdown CLI command.
* Tests: BOM handling, CRLF normalization, bullet formats, dedup logic,
Expand Down Expand Up @@ -416,6 +416,76 @@ describe("import-markdown CLI", () => {
);
});
});
describe("skip non-file .md entries", () => {
it("skips a directory named YYYY-MM-DD.md without aborting import", async () => {
const wsDir = await setupWorkspace("nonfile-test");
// Create memory/ subdirectory first
await mkdir(join(wsDir, "memory"), { recursive: true });
// Create a real .md file
await writeFile(
join(wsDir, "memory", "2026-04-11.md"),
"- Real file entry\n",
"utf-8",
);
// Create a directory that looks like a .md file (the bug scenario)
const fakeDir = join(wsDir, "memory", "2026-04-12.md");
await mkdir(fakeDir, { recursive: true });

const ctx = { embedder: mockEmbedder, store: mockStore };
let threw = false;
try {
const { imported, skipped } = await runImportMarkdown(ctx, {
openclawHome: testWorkspaceDir,
workspaceGlob: "nonfile-test",
});
// Should have imported the real file (1 entry from "- Real file entry")
assert.strictEqual(imported, 1, "should import the real .md file");
// skipped === 0: f.isFile() silently filters .md directories during mdFiles collection.
// This is correct — the directory doesn't cause EISDIR or increment skipped.
assert.strictEqual(skipped, 0, "directory silently filtered by f.isFile() — not counted as skipped");
} catch (err) {
threw = true;
throw new Error(`Import aborted on .md directory: ${err}`);
}
assert.ok(!threw, "import should not abort when encountering .md directory");
});

// Regression test for flatMemoryDir path (workspace/memory/YYYY-MM-DD.md)
// This path was missing withFileTypes: true in cli.ts, causing .md directories
// to be pushed to mdFiles and later causing EISDIR errors during readFile
it("skips a .md directory in flatMemoryDir without aborting import", async () => {
const wsDir = await setupWorkspace("flatmd-dir-test");
// Create memory/ subdirectory for flat structure
await mkdir(join(wsDir, "memory"), { recursive: true });
// Create a real .md file
await writeFile(
join(wsDir, "memory", "2026-04-11.md"),
"- Real flat file entry\n",
"utf-8",
);
// Create a directory that looks like a .md file in flat memory path
const fakeDir = join(wsDir, "memory", "2026-04-12.md");
await mkdir(fakeDir, { recursive: true });

const ctx = { embedder: mockEmbedder, store: mockStore };
let threw = false;
try {
// This specifically tests the flatMemoryDir path (no workspaceGlob)
const { imported, skipped } = await runImportMarkdown(ctx, {
openclawHome: testWorkspaceDir,
workspaceGlob: "flatmd-dir-test",
});
assert.strictEqual(imported, 1, "should import the real .md file");
// skipped === 0: f.isFile() in flatMemoryDir scan (cli.ts:639) silently filters
// .md directories during collection — no EISDIR error, no skipped++ increment.
assert.strictEqual(skipped, 0, "directory silently filtered by f.isFile() — not counted as skipped");
} catch (err) {
threw = true;
throw new Error(`Import aborted on .md directory in flatMemoryDir: ${err}`);
}
assert.ok(!threw, "import should not abort when encountering .md directory in flatMemoryDir");
});
});
});

// ────────────────────────────────────────────────────────────────────────────── Test runner helper ──────────────────────────────────────────────────────────────────────────────
Expand Down
Loading