perf(incremental-tokenize): checkpoint every 32 lines instead of every line (#481)

parseIncremental stored a full tokenizer snapshot after each line, so a
large editor document paid O(lines × state) memory and snapshot time on
first paint. Record checkpoints on a fixed interval (and always at the
document end), matching the density approach tokenized-document already
uses for windowed rendering. reparseIncremental keeps the same interval
when recording the dirty region so density does not grow back to one per
line after edits.

Author: ljodea

src/incremental-tokenize.js

@@ -94,8 +94,16 @@ function stateConverges(a, b) {
 }
 
 /**
- * Full parse with a line checkpoint after every line. First paint or language
- * change.
+ * Default gap between line checkpoints. Matches the spirit of
+ * `tokenized-document`'s interval: denser than its windowed default (100)
+ * so mid-document edits still resume nearby, sparse enough that a 10k-line
+ * file stores O(hundreds) of snapshots rather than O(lines).
+ */
+export const CHECKPOINT_INTERVAL = 32;
+
+/**
+ * Full parse with a line checkpoint every `CHECKPOINT_INTERVAL` lines (and
+ * always at the document end). First paint or language change.
  * @param {Registry} registry
  * @param {string} language
  * @param {string} code
@@ -104,8 +112,18 @@ function stateConverges(a, b) {
 export function parseIncremental(registry, language, code) {
   const session = registry.createSession(language);
   const checkpoints = [session.snapshot()];
+  let linesSinceCheckpoint = 0;
   for (const line of splitKeepEnds(code)) {
     session.append(line);
+    linesSinceCheckpoint++;
+    if (linesSinceCheckpoint >= CHECKPOINT_INTERVAL) {
+      checkpoints.push(session.snapshot());
+      linesSinceCheckpoint = 0;
+    }
+  }
+  // Always retain an end checkpoint so resume can land on the final state
+  // even when the last interval is incomplete.
+  if (linesSinceCheckpoint > 0 || checkpoints.length === 1) {
     checkpoints.push(session.snapshot());
   }
   const { events } = session.finish();
@@ -173,16 +191,19 @@ export function reparseIncremental(registry, language, previous, code) {
   const newLines = splitKeepEnds(code.slice(resumeCheckpoint.pos));
   let convergedAtOldIndex = -1;
   let oldIndex = resumeIndex;
+  let linesSinceCheckpoint = 0;
 
-  for (const line of newLines) {
-    session.append(line);
+  for (let li = 0; li < newLines.length; li++) {
+    session.append(/** @type {string} */ (newLines[li]));
     const snap = session.snapshot();
-    // snap.eventCount is session-local; shift to index the combined array.
-    checkpoints.push({
-      ...snap,
-      eventCount: snap.eventCount + prefixEvents.length,
-    });
-
+    linesSinceCheckpoint++;
+    // Check every line for convergence against previous checkpoints.
+    // Store every line for a window right after the edit (typing tends to
+    // stay near the cursor, so a follow-up edit here resumes in O(1)
+    // instead of walking to the next interval boundary), then fall back to
+    // the sparse interval so density doesn't stay O(lines) further out.
+    let shouldStore =
+      li < CHECKPOINT_INTERVAL || linesSinceCheckpoint >= CHECKPOINT_INTERVAL;
     if (snap.pos >= newSuffixStart) {
       const targetOldPos = snap.pos - posOffset;
       while (
@@ -201,9 +222,19 @@ export function reparseIncremental(registry, language, previous, code) {
         stateConverges(snap, oldCheckpoint)
       ) {
         convergedAtOldIndex = oldIndex;
-        break;
+        shouldStore = true;
       }
     }
+    if (!shouldStore && li === newLines.length - 1) shouldStore = true;
+    if (shouldStore) {
+      // snap.eventCount is session-local; shift to index the combined array.
+      checkpoints.push({
+        ...snap,
+        eventCount: snap.eventCount + prefixEvents.length,
+      });
+      linesSinceCheckpoint = 0;
+    }
+    if (convergedAtOldIndex >= 0) break;
   }
 
   if (convergedAtOldIndex >= 0) {

tests/e2e/e2e.test.ts

@@ -1538,12 +1538,19 @@ test("Typewriter - pausing then resuming continues without restarting", async ({
   mount,
   page,
 }) => {
+  // Under CI parallel load Firefox's setInterval can run well slower than
+  // `speed`, so a fixed 5s post-resume wait is enough to flake. Give the
+  // remaining ticks room, and poll for progress instead of sleeping.
+  test.setTimeout(20_000);
+
   await mount(Typewriter, { props: { speed: 15 } });
 
   const tw = page.getByTestId("tw");
   const revealed = tw.locator(".typewriter-unit:not(.typewriter-hidden)");
+  const done = page.getByTestId("done");
 
-  await page.waitForTimeout(120);
+  await expect.poll(async () => revealed.count()).toBeGreaterThan(0);
+  await expect(done).toHaveText("0");
   await page.getByTestId("toggle-play").click(); // pause
 
   const pausedCount = await revealed.count();
@@ -1555,7 +1562,8 @@ test("Typewriter - pausing then resuming continues without restarting", async ({
   expect(await revealed.count()).toBe(pausedCount);
 
   await page.getByTestId("toggle-play").click(); // resume
-  await expect(page.getByTestId("done")).toHaveText("1");
+  await expect.poll(async () => revealed.count()).toBeGreaterThan(pausedCount);
+  await expect(done).toHaveText("1", { timeout: 15_000 });
 
   // Same DOM node from before the pause: resume picked up where it left
   // off instead of re-tokenizing/rebuilding.

tests/incremental-tokenize.test.ts

@@ -5,6 +5,7 @@ import {
   toRanges,
 } from "../src/engine.js";
 import {
+  CHECKPOINT_INTERVAL,
   parseIncremental,
   reparseIncremental,
 } from "../src/incremental-tokenize.js";
@@ -254,3 +255,66 @@ describe("incremental re-tokenization survives a realistic mixed editing session
     ]);
   });
 });
+
+describe("parseIncremental checkpoint density", () => {
+  /** Build N one-statement lines of javascript. */
+  function manyLines(n: number): string {
+    let code = "";
+    for (let i = 0; i < n; i++) code += `const v${i} = ${i};\n`;
+    return code;
+  }
+
+  it("stores O(lines / interval) checkpoints, not one per line", () => {
+    const lineCount = 320;
+    const code = manyLines(lineCount);
+    const state = parseIncremental(registry, "javascript", code);
+    // Start checkpoint + one per interval + final (if not already on boundary).
+    // With interval 32: 320/32 = 10 interior boundaries → at most ~12 checkpoints,
+    // never one per line (321).
+    expect(state.checkpoints.length).toBeLessThan(lineCount / 4);
+    expect(state.checkpoints.length).toBeGreaterThan(2);
+    // Still ends at the document end so resume can reach the tail.
+    const last = state.checkpoints[state.checkpoints.length - 1];
+    expect(last?.pos).toBe(code.length);
+  });
+
+  it("reparse after a mid-document edit still matches a full re-parse", () => {
+    const code = manyLines(100);
+    const edited = code.replace("const v50 = 50;", "const v50 = 500;");
+    assertEditSequenceMatchesOneShot("javascript", [code, edited]);
+  });
+
+  it("follow-up edit near a prior edit converges in O(1) appended lines", () => {
+    const code = manyLines(320);
+    let state = parseIncremental(registry, "javascript", code);
+
+    const edit1 = code.replace("const v50 = 50;", "const v50 = 500;");
+    state = reparseIncremental(registry, "javascript", state, edit1);
+    expect(render(state.events)).toEqual(oneShotRender(edit1, "javascript"));
+
+    // First mid-doc edit densifies a pocket; the second edit on the same
+    // line must resume from that pocket and converge without walking a
+    // full CHECKPOINT_INTERVAL of sparse boundaries.
+    let appendCount = 0;
+    const createSession = registry.createSession.bind(registry);
+    registry.createSession = ((...args: Parameters<typeof createSession>) => {
+      const session = createSession(...args);
+      const append = session.append.bind(session);
+      session.append = (chunk: string) => {
+        appendCount++;
+        return append(chunk);
+      };
+      return session;
+    }) as typeof registry.createSession;
+
+    try {
+      const edit2 = edit1.replace("const v50 = 500;", "const v50 = 5000;");
+      state = reparseIncremental(registry, "javascript", state, edit2);
+      expect(render(state.events)).toEqual(oneShotRender(edit2, "javascript"));
+      expect(appendCount).toBeLessThan(CHECKPOINT_INTERVAL);
+      expect(appendCount).toBeLessThan(4);
+    } finally {
+      registry.createSession = createSession;
+    }
+  });
+});