Skip to content

Commit b646b16

Browse files
luandroluandro
committed
fix(notion-fetch): enforce stable sidebar ordering across incremental fetch-ready runs
1 parent 2b62949 commit b646b16

File tree

2 files changed

+319
-0
lines changed

2 files changed

+319
-0
lines changed

‎scripts/notion-fetch/generateBlocks.test.ts‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -841,6 +841,196 @@ describe("generateBlocks", () => {
841841
expect(content).toContain("sidebar_position: 12");
842842
});
843843

844+
it("should assign missing Order pages after the max existing sidebar_position in filtered runs", async () => {
845+
const { generateBlocks } = await import("./generateBlocks");
846+
const { PAGE_METADATA_CACHE_PATH } = await import("./pageMetadataCache");
847+
const mockWriteFileSync = fs.writeFileSync as Mock;
848+
849+
const page = createMockNotionPage({
850+
title: "New Incremental Page",
851+
elementType: "Page",
852+
});
853+
delete page.properties.Order;
854+
855+
const generateBlocksPath = fileURLToPath(
856+
new URL("./generateBlocks.ts", import.meta.url)
857+
);
858+
const generateBlocksDir = path.dirname(generateBlocksPath);
859+
const docsPath = path.join(generateBlocksDir, "../../docs");
860+
const existingDocPath = path.join(docsPath, "existing-order-page.md");
861+
const newDocPath = path.join(docsPath, "new-incremental-page.md");
862+
863+
fs.writeFileSync(
864+
existingDocPath,
865+
`---\nsidebar_position: "20"\n---\n\n# Existing Ordered Page\n`,
866+
"utf-8"
867+
);
868+
869+
fs.writeFileSync(
870+
PAGE_METADATA_CACHE_PATH,
871+
JSON.stringify({
872+
version: "1.0",
873+
scriptHash: "test-hash",
874+
lastSync: "2024-01-01T00:00:00.000Z",
875+
pages: {
876+
"existing-page-id": {
877+
lastEdited: "2024-01-01T00:00:00.000Z",
878+
outputPaths: ["/docs/existing-order-page.md"],
879+
processedAt: "2024-01-01T00:00:00.000Z",
880+
},
881+
},
882+
}),
883+
"utf-8"
884+
);
885+
886+
n2m.pageToMarkdown.mockResolvedValue([]);
887+
n2m.toMarkdownString.mockReturnValue({
888+
parent: "# New Incremental Page\n\nContent here.",
889+
});
890+
891+
await generateBlocks([page], vi.fn());
892+
893+
const markdownCalls = mockWriteFileSync.mock.calls.filter(
894+
(call) => typeof call[0] === "string" && call[0] === newDocPath
895+
);
896+
897+
expect(markdownCalls.length).toBeGreaterThan(0);
898+
899+
const content = markdownCalls[markdownCalls.length - 1][1] as string;
900+
expect(content).toContain("sidebar_position: 21");
901+
});
902+
903+
it("should assign unique sequential fallback positions for multiple missing Order pages", async () => {
904+
const { generateBlocks } = await import("./generateBlocks");
905+
const { PAGE_METADATA_CACHE_PATH } = await import("./pageMetadataCache");
906+
const mockWriteFileSync = fs.writeFileSync as Mock;
907+
908+
const pageOne = createMockNotionPage({
909+
title: "Missing One",
910+
elementType: "Page",
911+
});
912+
const pageTwo = createMockNotionPage({
913+
title: "Missing Two",
914+
elementType: "Page",
915+
});
916+
delete pageOne.properties.Order;
917+
delete pageTwo.properties.Order;
918+
919+
const generateBlocksPath = fileURLToPath(
920+
new URL("./generateBlocks.ts", import.meta.url)
921+
);
922+
const generateBlocksDir = path.dirname(generateBlocksPath);
923+
const docsPath = path.join(generateBlocksDir, "../../docs");
924+
const existingDocPath = path.join(docsPath, "existing-order-page.md");
925+
const firstDocPath = path.join(docsPath, "missing-one.md");
926+
const secondDocPath = path.join(docsPath, "missing-two.md");
927+
928+
fs.writeFileSync(
929+
existingDocPath,
930+
`---\nsidebar_position: "20"\n---\n\n# Existing Ordered Page\n`,
931+
"utf-8"
932+
);
933+
934+
fs.writeFileSync(
935+
PAGE_METADATA_CACHE_PATH,
936+
JSON.stringify({
937+
version: "1.0",
938+
scriptHash: "test-hash",
939+
lastSync: "2024-01-01T00:00:00.000Z",
940+
pages: {
941+
"existing-page-id": {
942+
lastEdited: "2024-01-01T00:00:00.000Z",
943+
outputPaths: ["/docs/existing-order-page.md"],
944+
processedAt: "2024-01-01T00:00:00.000Z",
945+
},
946+
},
947+
}),
948+
"utf-8"
949+
);
950+
951+
n2m.pageToMarkdown.mockResolvedValue([]);
952+
n2m.toMarkdownString.mockReturnValue({
953+
parent: "# Content\n\nBody.",
954+
});
955+
956+
await generateBlocks([pageOne, pageTwo], vi.fn());
957+
958+
const firstWrites = mockWriteFileSync.mock.calls.filter(
959+
(call) => typeof call[0] === "string" && call[0] === firstDocPath
960+
);
961+
const secondWrites = mockWriteFileSync.mock.calls.filter(
962+
(call) => typeof call[0] === "string" && call[0] === secondDocPath
963+
);
964+
965+
expect(firstWrites.length).toBeGreaterThan(0);
966+
expect(secondWrites.length).toBeGreaterThan(0);
967+
968+
const firstContent = firstWrites[firstWrites.length - 1][1] as string;
969+
const secondContent = secondWrites[secondWrites.length - 1][1] as string;
970+
expect(firstContent).toContain("sidebar_position: 21");
971+
expect(secondContent).toContain("sidebar_position: 22");
972+
});
973+
974+
it("should recover max sidebar_position from existing markdown when cache is missing", async () => {
975+
const { generateBlocks } = await import("./generateBlocks");
976+
const { PAGE_METADATA_CACHE_PATH } = await import("./pageMetadataCache");
977+
const mockWriteFileSync = fs.writeFileSync as Mock;
978+
const mockReaddirSync = fs.readdirSync as Mock;
979+
const mockUnlinkSync = fs.unlinkSync as Mock;
980+
981+
const page = createMockNotionPage({
982+
title: "Cacheless Incremental Page",
983+
elementType: "Page",
984+
});
985+
delete page.properties.Order;
986+
987+
const generateBlocksPath = fileURLToPath(
988+
new URL("./generateBlocks.ts", import.meta.url)
989+
);
990+
const generateBlocksDir = path.dirname(generateBlocksPath);
991+
const docsPath = path.join(generateBlocksDir, "../../docs");
992+
const existingDocPath = path.join(docsPath, "existing-disk-page.md");
993+
const newDocPath = path.join(docsPath, "cacheless-incremental-page.md");
994+
995+
fs.writeFileSync(
996+
existingDocPath,
997+
`---\nsidebar_position: "30"\n---\n\n# Existing Disk Page\n`,
998+
"utf-8"
999+
);
1000+
mockUnlinkSync(PAGE_METADATA_CACHE_PATH);
1001+
1002+
mockReaddirSync.mockImplementation(
1003+
(targetPath: string, options?: any) => {
1004+
if (targetPath === docsPath && options?.withFileTypes) {
1005+
return [
1006+
{
1007+
name: "existing-disk-page.md",
1008+
isDirectory: () => false,
1009+
isFile: () => true,
1010+
},
1011+
];
1012+
}
1013+
return [];
1014+
}
1015+
);
1016+
1017+
n2m.pageToMarkdown.mockResolvedValue([]);
1018+
n2m.toMarkdownString.mockReturnValue({
1019+
parent: "# Cacheless Incremental Page\n\nContent here.",
1020+
});
1021+
1022+
await generateBlocks([page], vi.fn());
1023+
1024+
const markdownCalls = mockWriteFileSync.mock.calls.filter(
1025+
(call) => typeof call[0] === "string" && call[0] === newDocPath
1026+
);
1027+
1028+
expect(markdownCalls.length).toBeGreaterThan(0);
1029+
1030+
const content = markdownCalls[markdownCalls.length - 1][1] as string;
1031+
expect(content).toContain("sidebar_position: 31");
1032+
});
1033+
8441034
it("should not reuse existing sidebar_position on full sync when Order is missing", async () => {
8451035
const { generateBlocks } = await import("./generateBlocks");
8461036
const mockWriteFileSync = fs.writeFileSync as Mock;

‎scripts/notion-fetch/generateBlocks.ts‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,86 @@ export function findExistingSidebarPosition(
251251
return null;
252252
}
253253

254+
function findMaxExistingSidebarPosition(
255+
metadataCache: PageMetadataCache,
256+
existingCache?: PageMetadataCache
257+
): number | null {
258+
const candidatePaths: string[] = [];
259+
const seenPaths = new Set<string>();
260+
let maxPosition: number | null = null;
261+
262+
const addCandidate = (candidate?: string) => {
263+
const resolvedPath = normalizePath(candidate ?? "");
264+
if (!resolvedPath || seenPaths.has(resolvedPath)) {
265+
return;
266+
}
267+
seenPaths.add(resolvedPath);
268+
candidatePaths.push(resolvedPath);
269+
};
270+
271+
const addCachePaths = (cache?: PageMetadataCache) => {
272+
if (!cache?.pages) {
273+
return;
274+
}
275+
276+
for (const metadata of Object.values(cache.pages)) {
277+
for (const outputPath of metadata.outputPaths ?? []) {
278+
addCandidate(outputPath);
279+
}
280+
}
281+
};
282+
283+
addCachePaths(metadataCache);
284+
addCachePaths(existingCache);
285+
286+
const addMarkdownFilesRecursively = (directoryPath: string) => {
287+
if (!directoryPath || !fs.existsSync(directoryPath)) {
288+
return;
289+
}
290+
291+
let entries: fs.Dirent[] = [];
292+
try {
293+
entries = fs.readdirSync(directoryPath, { withFileTypes: true });
294+
} catch {
295+
return;
296+
}
297+
298+
for (const entry of entries) {
299+
const entryPath = path.join(directoryPath, entry.name);
300+
if (entry.isDirectory()) {
301+
addMarkdownFilesRecursively(entryPath);
302+
continue;
303+
}
304+
305+
if (entry.isFile() && entry.name.endsWith(".md")) {
306+
addCandidate(entryPath);
307+
}
308+
}
309+
};
310+
311+
addMarkdownFilesRecursively(CONTENT_PATH);
312+
for (const locale of locales.filter((locale) => locale !== DEFAULT_LOCALE)) {
313+
addMarkdownFilesRecursively(getI18NPath(locale));
314+
}
315+
316+
for (const candidatePath of candidatePaths) {
317+
if (!fs.existsSync(candidatePath)) {
318+
continue;
319+
}
320+
321+
const content = fs.readFileSync(candidatePath, "utf-8");
322+
const position = extractSidebarPositionFromFrontmatter(content);
323+
if (position === null) {
324+
continue;
325+
}
326+
327+
maxPosition =
328+
maxPosition === null ? position : Math.max(maxPosition, position);
329+
}
330+
331+
return maxPosition;
332+
}
333+
254334
// setTranslationString moved to translationManager.ts
255335

256336
/**
@@ -743,6 +823,45 @@ export async function generateBlocks(
743823
return count + Object.keys(pageGroup.content).length;
744824
}, 0);
745825
let pageProcessingIndex = 0;
826+
const pageGroupSidebarPositions = new Map<number, number>();
827+
let nextGeneratedSidebarPosition: number | null = null;
828+
829+
let maxExplicitOrderInRun: number | null = null;
830+
for (const pageByLang of pagesByLang) {
831+
for (const page of Object.values(pageByLang.content ?? {})) {
832+
const orderValue = (page as any)?.properties?.["Order"]?.number;
833+
if (typeof orderValue !== "number" || !Number.isFinite(orderValue)) {
834+
continue;
835+
}
836+
maxExplicitOrderInRun =
837+
maxExplicitOrderInRun === null
838+
? orderValue
839+
: Math.max(maxExplicitOrderInRun, orderValue);
840+
}
841+
}
842+
843+
const getNextGeneratedSidebarPosition = () => {
844+
if (nextGeneratedSidebarPosition === null) {
845+
const maxKnownPosition = findMaxExistingSidebarPosition(
846+
metadataCache,
847+
existingCache ?? undefined
848+
);
849+
const baselineCandidates = [
850+
maxKnownPosition,
851+
maxExplicitOrderInRun,
852+
].filter(
853+
(value): value is number =>
854+
typeof value === "number" && Number.isFinite(value)
855+
);
856+
const baseline =
857+
baselineCandidates.length > 0 ? Math.max(...baselineCandidates) : 0;
858+
nextGeneratedSidebarPosition = baseline + 1;
859+
}
860+
861+
const position = nextGeneratedSidebarPosition;
862+
nextGeneratedSidebarPosition += 1;
863+
return position;
864+
};
746865

747866
const blocksMap = new Map<string, { key: string; data: any[] }>();
748867
const markdownMap = new Map<string, { key: string; data: any }>();
@@ -873,6 +992,10 @@ export async function generateBlocks(
873992
const orderValue = props?.["Order"]?.number;
874993
// Fix: Use !== undefined check instead of Number.isFinite to properly handle 0 values
875994
let sidebarPosition = orderValue !== undefined ? orderValue : null;
995+
const groupSidebarPosition = pageGroupSidebarPositions.get(i);
996+
if (sidebarPosition === null && groupSidebarPosition !== undefined) {
997+
sidebarPosition = groupSidebarPosition;
998+
}
876999
if (sidebarPosition === null && !enableDeletion) {
8771000
sidebarPosition = findExistingSidebarPosition(
8781001
page.id,
@@ -882,9 +1005,15 @@ export async function generateBlocks(
8821005
syncMode.fullRebuild
8831006
);
8841007
}
1008+
if (sidebarPosition === null && !enableDeletion) {
1009+
sidebarPosition = getNextGeneratedSidebarPosition();
1010+
}
8851011
if (sidebarPosition === null) {
8861012
sidebarPosition = i + 1;
8871013
}
1014+
if (!pageGroupSidebarPositions.has(i)) {
1015+
pageGroupSidebarPositions.set(i, sidebarPosition);
1016+
}
8881017

8891018
const customProps: Record<string, unknown> = {};
8901019
if (

0 commit comments

Comments
 (0)