feat: define paper ingestion coverage contracts

Author: MrFaruk0

docs/HANDOFF.md

@@ -1,5 +1,17 @@
 # Development handoff
 
+## 2026-08-31 First paper-substrate contract increment
+
+Completed the strict, versioned contracts for paper sources, UTF-8 locators, sections, chunks,
+revisions, extraction-versus-processing coverage, and cache identity. Extraction regions partition
+the source, and chunks partition extracted regions. Processing is complete only when extraction is
+complete and all chunks are processed. Portable path safety is covered. No parser, persistence,
+model analysis, PDF support, or `/paper` command exists yet.
+
+Validation: contracts typecheck; 26/26 contracts tests; Biome on three files; and
+`git diff --check` passed. Next: deterministic local txt/md/tex research ingestion, then the
+Harness/CLI `/paper` vertical slice.
+
 ## 2026-08-31 Phase 2 stabilization addendum
 
 Independent review and re-review are clean and commit-ready after fixes. The fixes addressed exact

packages/contracts/src/index.ts

@@ -2,6 +2,7 @@ export * from "./errors.js";
 export * from "./events.js";
 export * from "./ids.js";
 export * from "./model.js";
+export * from "./paper.js";
 export * from "./run.js";
 export * from "./runtime-protocol.js";
 export * from "./tools.js";

packages/contracts/src/paper.ts

@@ -0,0 +1,340 @@
+import { z } from "zod";
+import {
+  ContractSchemaVersionSchema,
+  EntityIdSchema,
+  SafeTextSchema,
+  Sha256Schema,
+} from "./ids.js";
+
+const RevisionSchema = z
+  .string()
+  .min(1)
+  .max(128)
+  .regex(/^[A-Za-z0-9][A-Za-z0-9._+/-]*$/u, "Invalid revision identifier");
+const RelativePathSchema = z
+  .string()
+  .min(1)
+  .max(1024)
+  .refine((value) => !/^[A-Za-z]:|^[/\\]/u.test(value), "Absolute and drive paths are not allowed")
+  .refine((value) => value === value.trim(), "Leading and trailing whitespace are not allowed")
+  .refine(
+    (value) =>
+      value
+        .split("/")
+        .every(
+          (part) =>
+            part.length > 0 &&
+            part.length <= 255 &&
+            part !== "." &&
+            part !== ".." &&
+            !/[. ]$/u.test(part) &&
+            !/[<>:"\\|?*]/u.test(part) &&
+            !/^(?:CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(?:\..*)?$/iu.test(part),
+        ),
+    "Invalid relative path",
+  )
+  .refine(
+    (value) =>
+      [...value].every((character) => {
+        const code = character.codePointAt(0) ?? 0;
+        return code > 0x1f && (code < 0x7f || code > 0x9f);
+      }),
+    "Control characters are not allowed",
+  );
+
+export const PaperFormatSchema = z.enum(["text", "markdown", "latex"]);
+export type PaperFormat = z.infer<typeof PaperFormatSchema>;
+
+export const PaperSourceSnapshotSchema = z
+  .object({
+    sourceId: EntityIdSchema,
+    relativePath: RelativePathSchema,
+    format: PaperFormatSchema,
+    mediaType: z
+      .string()
+      .min(1)
+      .max(128)
+      .regex(/^[\w.+-]+\/[\w.+-]+$/u),
+    byteLength: z.number().int().positive().max(1_000_000_000),
+    contentHash: Sha256Schema,
+  })
+  .strict()
+  .readonly();
+export type PaperSourceSnapshot = z.infer<typeof PaperSourceSnapshotSchema>;
+
+export const PaperByteRangeSchema = z
+  .object({ startByte: z.number().int().nonnegative(), endByte: z.number().int().nonnegative() })
+  .strict()
+  .superRefine((range, context) => {
+    if (range.endByte <= range.startByte)
+      context.addIssue({ code: "custom", message: "Range must be non-empty" });
+  })
+  .readonly();
+export type PaperByteRange = z.infer<typeof PaperByteRangeSchema>;
+
+export const PaperLocatorSchema = z
+  .object({
+    startByte: z.number().int().nonnegative(),
+    endByte: z.number().int().nonnegative(),
+    startLine: z.number().int().positive(),
+    endLine: z.number().int().positive(),
+    sectionPath: z.array(SafeTextSchema).max(32).readonly().optional(),
+  })
+  .strict()
+  .superRefine((locator, context) => {
+    if (locator.endByte <= locator.startByte)
+      context.addIssue({ code: "custom", message: "Locator must be non-empty" });
+    if (locator.endLine < locator.startLine)
+      context.addIssue({ code: "custom", message: "Invalid line range" });
+  })
+  .readonly();
+export type PaperLocator = z.infer<typeof PaperLocatorSchema>;
+
+export const PaperSectionSchema = z
+  .object({
+    sectionId: EntityIdSchema,
+    title: SafeTextSchema,
+    level: z.number().int().positive().max(64),
+    locator: PaperLocatorSchema,
+  })
+  .strict()
+  .readonly();
+export type PaperSection = z.infer<typeof PaperSectionSchema>;
+
+export const PaperChunkSchema = z
+  .object({
+    chunkId: EntityIdSchema,
+    index: z.number().int().nonnegative(),
+    locator: PaperLocatorSchema,
+    contentHash: Sha256Schema,
+    content: z.string().min(1).max(16_000_000),
+    estimatedTokenCount: z.number().int().nonnegative().max(16_000_000),
+  })
+  .strict()
+  .superRefine((chunk, context) => {
+    if (
+      new TextEncoder().encode(chunk.content).byteLength !==
+      chunk.locator.endByte - chunk.locator.startByte
+    ) {
+      context.addIssue({
+        code: "custom",
+        message: "Chunk content length must match its byte locator",
+      });
+    }
+  })
+  .readonly();
+export type PaperChunk = z.infer<typeof PaperChunkSchema>;
+
+export const PaperRevisionsSchema = z
+  .object({ extractor: RevisionSchema, chunker: RevisionSchema, tokenEstimator: RevisionSchema })
+  .strict()
+  .readonly();
+export type PaperRevisions = z.infer<typeof PaperRevisionsSchema>;
+
+export const PaperExtractionCoverageSchema = z
+  .object({
+    complete: z.boolean(),
+    extractedRegions: z.array(PaperByteRangeSchema).max(1_000_000).readonly(),
+    omittedRegions: z
+      .array(z.object({ range: PaperByteRangeSchema, reason: SafeTextSchema }).strict().readonly())
+      .max(1_000_000)
+      .readonly(),
+  })
+  .strict()
+  .readonly();
+export type PaperExtractionCoverage = z.infer<typeof PaperExtractionCoverageSchema>;
+
+export const PaperProcessingStateSchema = z.enum([
+  "not_requested",
+  "processed",
+  "failed",
+  "skipped",
+]);
+export type PaperProcessingState = z.infer<typeof PaperProcessingStateSchema>;
+
+const ProcessingCountsSchema = z
+  .object({
+    notRequested: z.number().int().nonnegative(),
+    processed: z.number().int().nonnegative(),
+    failed: z.number().int().nonnegative(),
+    skipped: z.number().int().nonnegative(),
+  })
+  .strict()
+  .readonly();
+
+export const PaperProcessingEntrySchema = z
+  .object({ chunkId: EntityIdSchema, state: PaperProcessingStateSchema })
+  .strict()
+  .readonly();
+
+export const PaperProcessingCoverageSchema = z
+  .object({
+    complete: z.boolean(),
+    entries: z.array(PaperProcessingEntrySchema).max(1_000_000).readonly(),
+    counts: ProcessingCountsSchema,
+  })
+  .strict()
+  .readonly();
+export type PaperProcessingCoverage = z.infer<typeof PaperProcessingCoverageSchema>;
+
+export const PaperIngestionManifestSchema = z
+  .object({
+    schemaVersion: ContractSchemaVersionSchema,
+    cacheIdentity: Sha256Schema,
+    sourceSnapshot: PaperSourceSnapshotSchema,
+    revisions: PaperRevisionsSchema,
+    sections: z.array(PaperSectionSchema).max(100_000).readonly(),
+    chunks: z.array(PaperChunkSchema).max(1_000_000).readonly(),
+    extraction: PaperExtractionCoverageSchema,
+    processing: PaperProcessingCoverageSchema,
+  })
+  .strict()
+  .superRefine((manifest, context) => {
+    const { chunks, sourceSnapshot, extraction, processing } = manifest;
+    let previousEnd = 0;
+    const ids = new Set<string>();
+    for (const [index, chunk] of chunks.entries()) {
+      if (chunk.index !== index)
+        context.addIssue({
+          code: "custom",
+          path: ["chunks", index, "index"],
+          message: "Chunk indexes must be contiguous from zero",
+        });
+      if (ids.has(chunk.chunkId))
+        context.addIssue({
+          code: "custom",
+          path: ["chunks", index, "chunkId"],
+          message: "Chunk IDs must be unique",
+        });
+      ids.add(chunk.chunkId);
+      if (
+        chunk.locator.startByte < previousEnd ||
+        chunk.locator.endByte > sourceSnapshot.byteLength
+      )
+        context.addIssue({
+          code: "custom",
+          path: ["chunks", index, "locator"],
+          message: "Chunk locators must be ordered and within source",
+        });
+      previousEnd = chunk.locator.endByte;
+    }
+    const extracted = extraction.extractedRegions;
+    const omitted = extraction.omittedRegions.map(({ range }) => range);
+    for (const ranges of [extracted, omitted]) {
+      for (let index = 1; index < ranges.length; index++) {
+        const current = ranges[index];
+        const previous = ranges[index - 1];
+        if (current !== undefined && previous !== undefined && current.startByte < previous.endByte)
+          context.addIssue({
+            code: "custom",
+            path: ["extraction"],
+            message: "Regions must be ordered and non-overlapping",
+          });
+      }
+    }
+    let ei = 0;
+    let oi = 0;
+    let cursor = 0;
+    while (ei < extracted.length || oi < omitted.length) {
+      const e = extracted[ei];
+      const o = omitted[oi];
+      const next = o === undefined || (e !== undefined && e.startByte < o.startByte) ? e : o;
+      if (next === undefined || next.startByte !== cursor) break;
+      cursor = next.endByte;
+      if (next === e) ei++;
+      else oi++;
+    }
+    if (cursor !== sourceSnapshot.byteLength || ei !== extracted.length || oi !== omitted.length)
+      context.addIssue({
+        code: "custom",
+        path: ["extraction"],
+        message: "Extraction ranges contain gaps, overlaps, or exceed source",
+      });
+    if (extraction.complete !== (omitted.length === 0))
+      context.addIssue({
+        code: "custom",
+        path: ["extraction", "complete"],
+        message: "Extraction completeness disagrees with omitted regions",
+      });
+    let ri = 0;
+    let chunkCursor = extracted[0]?.startByte;
+    for (const chunk of chunks) {
+      while (ri < extracted.length && chunkCursor === extracted[ri]?.endByte) {
+        ri++;
+        chunkCursor = extracted[ri]?.startByte;
+      }
+      const region = extracted[ri];
+      if (
+        chunkCursor === undefined ||
+        chunk.locator.startByte !== chunkCursor ||
+        region === undefined ||
+        chunk.locator.endByte > region.endByte
+      )
+        context.addIssue({
+          code: "custom",
+          path: ["chunks"],
+          message: "Chunks must exactly partition extracted regions",
+        });
+      chunkCursor = chunk.locator.endByte;
+    }
+    while (ri < extracted.length && chunkCursor === extracted[ri]?.endByte) {
+      ri++;
+      chunkCursor = extracted[ri]?.startByte;
+    }
+    if (ri !== extracted.length)
+      context.addIssue({
+        code: "custom",
+        path: ["chunks"],
+        message: "Chunks must consume every extracted region",
+      });
+    const sectionIds = new Set<string>();
+    for (const [index, section] of manifest.sections.entries()) {
+      if (sectionIds.has(section.sectionId) || section.locator.endByte > sourceSnapshot.byteLength)
+        context.addIssue({
+          code: "custom",
+          path: ["sections", index],
+          message: "Section IDs and locators must be unique and bounded",
+        });
+      sectionIds.add(section.sectionId);
+    }
+    const entryIds = new Set<string>();
+    const counts: Record<"notRequested" | "processed" | "failed" | "skipped", number> = {
+      notRequested: 0,
+      processed: 0,
+      failed: 0,
+      skipped: 0,
+    };
+    for (const entry of processing.entries) {
+      if (
+        entryIds.has(entry.chunkId) ||
+        !ids.has(entry.chunkId) ||
+        entry.chunkId !== chunks[entryIds.size]?.chunkId
+      )
+        context.addIssue({
+          code: "custom",
+          path: ["processing"],
+          message: "Processing entries must correspond exactly once to chunks",
+        });
+      entryIds.add(entry.chunkId);
+      const countKey = entry.state === "not_requested" ? "notRequested" : entry.state;
+      counts[countKey]++;
+    }
+    if (
+      entryIds.size !== chunks.length ||
+      Object.entries(counts).some(
+        ([key, value]) => processing.counts[key as keyof typeof counts] !== value,
+      ) ||
+      processing.complete !==
+        (extraction.complete && chunks.length > 0 && counts.processed === chunks.length)
+    )
+      context.addIssue({
+        code: "custom",
+        path: ["processing"],
+        message: "Processing summary disagrees with entries",
+      });
+  })
+  .readonly();
+export type PaperIngestionManifest = z.infer<typeof PaperIngestionManifestSchema>;
+
+export const PaperManifestSchema = PaperIngestionManifestSchema;
+export type PaperManifest = PaperIngestionManifest;