diff --git a/ts-packages/rust-qmd-json/.gitignore b/ts-packages/rust-qmd-json/.gitignore new file mode 100644 index 0000000..dd6e803 --- /dev/null +++ b/ts-packages/rust-qmd-json/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +dist/ +*.log +.DS_Store diff --git a/ts-packages/rust-qmd-json/README.md b/ts-packages/rust-qmd-json/README.md new file mode 100644 index 0000000..38b4d84 --- /dev/null +++ b/ts-packages/rust-qmd-json/README.md @@ -0,0 +1,138 @@ +# @quarto/rust-qmd-json + +Convert quarto-markdown-pandoc JSON output to AnnotatedParse structures with full source mapping. + +## Overview + +This package converts the JSON output from the Rust-based `quarto-markdown-pandoc` parser +into `AnnotatedParse` structures that are compatible with quarto-cli's YAML validation +infrastructure. It preserves complete source location information through the conversion. + +## Installation + +```bash +npm install @quarto/rust-qmd-json +``` + +## Quick Start + +```typescript +import { parseRustQmdMetadata } from '@quarto/rust-qmd-json'; +import type { RustQmdJson } from '@quarto/rust-qmd-json'; + +// JSON from quarto-markdown-pandoc +const json: RustQmdJson = { + meta: { + title: { t: 'MetaString', c: 'My Document', s: 0 } + }, + blocks: [], + source_pool: [ + { r: [11, 22], t: 0, d: 0 } + ], + source_context: { + files: [ + { id: 0, path: 'doc.qmd', content: '---\ntitle: My Document\n---' } + ] + } +}; + +const annotatedParse = parseRustQmdMetadata(json); + +console.log(annotatedParse.result); // { title: 'My Document' } +console.log(annotatedParse.kind); // 'mapping' +console.log(annotatedParse.components.length); // 2 (key + value) +``` + +## API + +### `parseRustQmdMetadata(json, errorHandler?)` + +Main entry point for converting quarto-markdown-pandoc JSON to AnnotatedParse. + +**Parameters:** +- `json: RustQmdJson` - The JSON output from quarto-markdown-pandoc +- `errorHandler?: (msg: string, id?: number) => void` - Optional error handler for SourceInfo reconstruction errors + +**Returns:** `AnnotatedParse` + +**Example with error handling:** + +```typescript +import { parseRustQmdMetadata } from '@quarto/rust-qmd-json'; + +const errorHandler = (msg: string, id?: number) => { + console.error(`SourceInfo error: ${msg}`, id); +}; + +const result = parseRustQmdMetadata(json, errorHandler); +``` + +### Types + +The package exports all necessary TypeScript types: + +```typescript +import type { + AnnotatedParse, + JSONValue, + JsonMetaValue, + MetaMapEntry, + RustQmdJson, + SerializableSourceInfo, + SourceContext, + SourceInfoErrorHandler +} from '@quarto/rust-qmd-json'; +``` + +### Advanced Usage + +For more control, you can use the underlying classes directly: + +```typescript +import { SourceInfoReconstructor, MetadataConverter } from '@quarto/rust-qmd-json'; + +const reconstructor = new SourceInfoReconstructor( + json.source_pool, + json.source_context +); + +const converter = new MetadataConverter(reconstructor); +const result = converter.convertMeta(json.meta); +``` + +## Development + +```bash +# Install dependencies +npm install + +# Build +npm run build + +# Test +npm test + +# Clean +npm run clean +``` + +## Architecture + +The conversion happens in two phases: + +1. **SourceInfo Reconstruction**: Convert the pooled SourceInfo format from JSON into + MappedString objects that track source locations through transformation chains. + +2. **Metadata Conversion**: Recursively convert MetaValue variants into AnnotatedParse + structures with proper source tracking. MetaInlines/MetaBlocks are treated as leaf + nodes with the JSON array structure preserved in the result. + +## Design Decisions + +- **Direct JSON Value Mapping**: MetaInlines and MetaBlocks are preserved as JSON arrays + in the `result` field, avoiding any text reconstruction +- **Source Tracking**: Every value can be traced back to original file location via SourceInfo +- **Compatible Types**: Produces AnnotatedParse structures compatible with existing validation code + +See repository's `claude-notes/plans/2025-10-23-json-to-annotated-parse-conversion.md` for +detailed implementation plan. diff --git a/ts-packages/rust-qmd-json/SETUP-NOTES.md b/ts-packages/rust-qmd-json/SETUP-NOTES.md new file mode 100644 index 0000000..1317415 --- /dev/null +++ b/ts-packages/rust-qmd-json/SETUP-NOTES.md @@ -0,0 +1,49 @@ +# Setup Notes + +## TypeScript Infrastructure Verified + +✅ Successfully set up standalone TypeScript package +✅ `@quarto/mapped-string` integration working +✅ All tests passing + +## Test Results + +``` +✔ can import and use @quarto/mapped-string (0.550583ms) +✔ can create mapped substrings (0.106792ms) +✔ placeholder conversion function (15.146917ms) +ℹ tests 3 +ℹ suites 0 +ℹ pass 3 +ℹ fail 0 +``` + +## Environment + +- Node.js: v23.11.0 +- npm: 10.9.2 +- TypeScript: ^5.4.2 +- Dependencies: + - `@quarto/mapped-string`: ^0.1.8 (working correctly) + - `tsx`: ^4.7.1 (for running TypeScript tests) + - `@types/node`: ^20.0.0 + +## Key Learnings + +1. **Module imports**: @quarto/mapped-string exports `MappedString` as a type, not a value. + Must use `export type { MappedString }` in TypeScript. + +2. **Test framework**: Using Node.js built-in test runner with tsx for TypeScript execution. + Works well for ES modules. + +3. **Project structure**: Following Rust workspace conventions by placing TypeScript packages + in `ts-packages/` directory parallel to `crates/`. + +## Next Steps + +Ready to implement: +1. Phase 1: SourceInfo reconstruction +2. Phase 2: Metadata conversion +3. Phase 3: Integration & testing + +See `claude-notes/plans/2025-10-23-json-to-annotated-parse-conversion.md` for detailed plan. diff --git a/ts-packages/rust-qmd-json/package-lock.json b/ts-packages/rust-qmd-json/package-lock.json new file mode 100644 index 0000000..2456082 --- /dev/null +++ b/ts-packages/rust-qmd-json/package-lock.json @@ -0,0 +1,227 @@ +{ + "name": "@quarto/rust-qmd-json", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@quarto/rust-qmd-json", + "version": "0.1.0", + "license": "MIT", + "dependencies": { + "@quarto/mapped-string": "^0.1.8" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "tsx": "^4.7.1", + "typescript": "^5.4.2" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.11", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.11.tgz", + "integrity": "sha512-VekY0PBCukppoQrycFxUqkCojnTQhdec0vevUL/EDOCnXd9LKWqD/bHwMPzigIJXPhC59Vd1WFIL57SKs2mg4w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@quarto/mapped-string": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/@quarto/mapped-string/-/mapped-string-0.1.8.tgz", + "integrity": "sha512-NkHKvyola1Gw9RvI6JhOT6kvFx0HXgzXOay2LlF2gA09VkASCYaDaeWa5jME+c27tdBZ95IUueSAYFroJyrTJQ==", + "license": "MIT", + "dependencies": { + "@quarto/tidyverse-errors": "^0.1.9", + "ansi-colors": "^4.1.3", + "tsconfig": "*", + "typescript": "^5.4.2" + } + }, + "node_modules/@quarto/tidyverse-errors": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/@quarto/tidyverse-errors/-/tidyverse-errors-0.1.9.tgz", + "integrity": "sha512-JWA/teFA0XOv1UbAmNPX8bymBes/U0o9KNbvY0Aw1Mg7wY+vFRaVFWOicQuO6HrXtVM/6Osyy7IFY0KfKndy5w==", + "license": "MIT", + "dependencies": { + "ansi-colors": "^4.1.3", + "tsconfig": "*", + "typescript": "^5.4.2" + } + }, + "node_modules/@types/node": { + "version": "20.19.23", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.23.tgz", + "integrity": "sha512-yIdlVVVHXpmqRhtyovZAcSy0MiPcYWGkoO4CGe/+jpP0hmNuihm4XhHbADpK++MsiLHP5MVlv+bcgdF99kSiFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-xevGOReSYGM7g/kUBZzPqCrR/KYAo+F0yiPc85WFTJa0MSLtyFTVTU6cJu/aV4mid7IffDIWqo69THF2o4JiEQ==", + "license": "MIT" + }, + "node_modules/@types/strip-json-comments": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/@types/strip-json-comments/-/strip-json-comments-0.0.30.tgz", + "integrity": "sha512-7NQmHra/JILCd1QqpSzl8+mJRc8ZHz3uDm8YV1Ks9IhK0epEiTw8aIErbvH9PI+6XbqhyIQy3462nEsn7UVzjQ==", + "license": "MIT" + }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/esbuild": { + "version": "0.25.11", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.11.tgz", + "integrity": "sha512-KohQwyzrKTQmhXDW1PjCv3Tyspn9n5GcY2RTDqeORIdIJY8yKIF7sTSopFmn/wpMPW4rdPXI0UE5LJLuq3bx0Q==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.11", + "@esbuild/android-arm": "0.25.11", + "@esbuild/android-arm64": "0.25.11", + "@esbuild/android-x64": "0.25.11", + "@esbuild/darwin-arm64": "0.25.11", + "@esbuild/darwin-x64": "0.25.11", + "@esbuild/freebsd-arm64": "0.25.11", + "@esbuild/freebsd-x64": "0.25.11", + "@esbuild/linux-arm": "0.25.11", + "@esbuild/linux-arm64": "0.25.11", + "@esbuild/linux-ia32": "0.25.11", + "@esbuild/linux-loong64": "0.25.11", + "@esbuild/linux-mips64el": "0.25.11", + "@esbuild/linux-ppc64": "0.25.11", + "@esbuild/linux-riscv64": "0.25.11", + "@esbuild/linux-s390x": "0.25.11", + "@esbuild/linux-x64": "0.25.11", + "@esbuild/netbsd-arm64": "0.25.11", + "@esbuild/netbsd-x64": "0.25.11", + "@esbuild/openbsd-arm64": "0.25.11", + "@esbuild/openbsd-x64": "0.25.11", + "@esbuild/openharmony-arm64": "0.25.11", + "@esbuild/sunos-x64": "0.25.11", + "@esbuild/win32-arm64": "0.25.11", + "@esbuild/win32-ia32": "0.25.11", + "@esbuild/win32-x64": "0.25.11" + } + }, + "node_modules/get-tsconfig": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz", + "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tsconfig": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tsconfig/-/tsconfig-7.0.0.tgz", + "integrity": "sha512-vZXmzPrL+EmC4T/4rVlT2jNVMWCi/O4DIiSj3UHg1OE5kCKbk4mfrXc6dZksLgRM/TZlKnousKH9bbTazUWRRw==", + "license": "MIT", + "dependencies": { + "@types/strip-bom": "^3.0.0", + "@types/strip-json-comments": "0.0.30", + "strip-bom": "^3.0.0", + "strip-json-comments": "^2.0.0" + } + }, + "node_modules/tsx": { + "version": "4.20.6", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz", + "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.25.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/ts-packages/rust-qmd-json/package.json b/ts-packages/rust-qmd-json/package.json new file mode 100644 index 0000000..176a08f --- /dev/null +++ b/ts-packages/rust-qmd-json/package.json @@ -0,0 +1,36 @@ +{ + "name": "@quarto/rust-qmd-json", + "version": "0.1.0", + "description": "Convert quarto-markdown-pandoc JSON output to AnnotatedParse structures", + "license": "MIT", + "author": { + "name": "Posit PBC" + }, + "publishConfig": { + "access": "public", + "registry": "https://registry.npmjs.org/" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/quarto-dev/quarto.git", + "directory": "ts-packages/rust-qmd-json" + }, + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": ["dist"], + "scripts": { + "build": "tsc", + "test": "node --import tsx --test test/*.test.ts", + "clean": "rm -rf dist", + "prepublishOnly": "npm run clean && npm run build && npm test" + }, + "dependencies": { + "@quarto/mapped-string": "^0.1.8" + }, + "devDependencies": { + "typescript": "^5.4.2", + "tsx": "^4.7.1", + "@types/node": "^20.0.0" + } +} diff --git a/ts-packages/rust-qmd-json/src/index.ts b/ts-packages/rust-qmd-json/src/index.ts new file mode 100644 index 0000000..bdb139a --- /dev/null +++ b/ts-packages/rust-qmd-json/src/index.ts @@ -0,0 +1,83 @@ +/** + * @quarto/rust-qmd-json + * + * Converts quarto-markdown-pandoc JSON output to AnnotatedParse structures + * compatible with quarto-cli's YAML validation infrastructure. + */ + +// Re-export types and functions from @quarto/mapped-string +export type { MappedString } from '@quarto/mapped-string'; +export { asMappedString, mappedSubstring } from '@quarto/mapped-string'; + +// Re-export our types +export type { + AnnotatedParse, + JSONValue, + JsonMetaValue, + MetaMapEntry, + RustQmdJson +} from './types.js'; + +export type { + SerializableSourceInfo, + SourceContext, + SourceInfoErrorHandler +} from './source-map.js'; + +// Re-export classes +export { SourceInfoReconstructor } from './source-map.js'; +export { MetadataConverter } from './meta-converter.js'; + +// Import for main function +import { SourceInfoReconstructor } from './source-map.js'; +import { MetadataConverter } from './meta-converter.js'; +import type { RustQmdJson, AnnotatedParse } from './types.js'; +import type { SourceInfoErrorHandler } from './source-map.js'; + +/** + * Convert quarto-markdown-pandoc JSON output to AnnotatedParse + * + * @param json - The JSON output from quarto-markdown-pandoc + * @param errorHandler - Optional error handler for SourceInfo reconstruction errors + * @returns AnnotatedParse structure compatible with quarto-cli + * + * @example + * ```typescript + * import { parseRustQmdMetadata } from '@quarto/rust-qmd-json'; + * + * const json = { + * meta: { + * title: { t: 'MetaString', c: 'Hello', s: 0 } + * }, + * blocks: [], + * source_pool: [ + * { r: [11, 16], t: 0, d: 0 } + * ], + * source_context: { + * files: [ + * { id: 0, path: 'test.qmd', content: '---\ntitle: Hello\n---' } + * ] + * } + * }; + * + * const metadata = parseRustQmdMetadata(json); + * console.log(metadata.result); // { title: 'Hello' } + * ``` + */ +export function parseRustQmdMetadata( + json: RustQmdJson, + errorHandler?: SourceInfoErrorHandler +): AnnotatedParse { + // 1. Create SourceInfoReconstructor with pool and context + const sourceReconstructor = new SourceInfoReconstructor( + json.source_pool, + json.source_context, + errorHandler + ); + + // 2. Create MetadataConverter + const converter = new MetadataConverter(sourceReconstructor); + + // 3. Convert metadata to AnnotatedParse + return converter.convertMeta(json.meta); +} diff --git a/ts-packages/rust-qmd-json/src/meta-converter.ts b/ts-packages/rust-qmd-json/src/meta-converter.ts new file mode 100644 index 0000000..292818c --- /dev/null +++ b/ts-packages/rust-qmd-json/src/meta-converter.ts @@ -0,0 +1,315 @@ +/** + * Metadata Conversion + * + * Converts MetaValue structures from quarto-markdown-pandoc JSON + * into AnnotatedParse structures compatible with quarto-cli. + */ + +import type { AnnotatedParse, JsonMetaValue, MetaMapEntry, JSONValue } from './types.js'; +import type { SourceInfoReconstructor } from './source-map.js'; + +/** + * Type guard for MetaMap content structure + */ +function isMetaMapContent(c: unknown): c is { entries: MetaMapEntry[] } { + return ( + typeof c === 'object' && + c !== null && + 'entries' in c && + Array.isArray((c as { entries: unknown }).entries) + ); +} + +/** + * Type guard for checking if value is an array of JsonMetaValue + */ +function isMetaValueArray(c: unknown): c is JsonMetaValue[] { + return Array.isArray(c); +} + +/** + * Type guard for Span structure with yaml-tagged-string + */ +function isTaggedSpan(obj: unknown): obj is { + t: string; + c: [{ c: unknown; kv?: [string, string][] }, unknown]; +} { + return ( + typeof obj === 'object' && + obj !== null && + 't' in obj && + (obj as { t: unknown }).t === 'Span' && + 'c' in obj && + Array.isArray((obj as { c: unknown }).c) && + (obj as { c: unknown[] }).c.length === 2 + ); +} + +/** + * Converts metadata from quarto-markdown-pandoc JSON to AnnotatedParse + */ +export class MetadataConverter { + constructor(private sourceReconstructor: SourceInfoReconstructor) {} + + /** + * Convert top-level metadata object to AnnotatedParse + */ + convertMeta(jsonMeta: Record): AnnotatedParse { + // Create a synthetic MetaMap for the top-level metadata + const entries: MetaMapEntry[] = Object.entries(jsonMeta).map(([key, value]) => ({ + key, + key_source: value.s, // Use value's source for key (not ideal, but metadata doesn't include key sources) + value + })); + + // Find the overall range by getting min/max offsets + let minStart = Infinity; + let maxEnd = -Infinity; + for (const value of Object.values(jsonMeta)) { + const [start, end] = this.sourceReconstructor.getOffsets(value.s); + minStart = Math.min(minStart, start); + maxEnd = Math.max(maxEnd, end); + } + + // If no metadata, use defaults + if (minStart === Infinity) { + minStart = 0; + maxEnd = 0; + } + + // Convert to AnnotatedParse components + const components: AnnotatedParse[] = []; + const result: Record = {}; + + for (const [key, value] of Object.entries(jsonMeta)) { + // Create AnnotatedParse for key + const [keyStart, keyEnd] = this.sourceReconstructor.getOffsets(value.s); + const keySource = this.sourceReconstructor.toMappedString(value.s); + + const keyAP: AnnotatedParse = { + result: key, + kind: 'key', + source: keySource, + components: [], + start: keyStart, + end: keyEnd + }; + + // Create AnnotatedParse for value + const valueAP = this.convertMetaValue(value); + + // Interleave key and value + components.push(keyAP, valueAP); + result[key] = valueAP.result; + } + + // Create synthetic MappedString for top-level (using first file if available) + const firstFile = this.sourceReconstructor['sourceContext'].files[0]; + const topSource = this.sourceReconstructor['sourceContext'].files[0] + ? this.sourceReconstructor.toMappedString(0) // Use first SourceInfo if available + : this.sourceReconstructor.toMappedString(0); + + return { + result, + kind: 'mapping', // Top-level metadata is a mapping + source: topSource, + components, + start: minStart, + end: maxEnd + }; + } + + /** + * Convert individual MetaValue to AnnotatedParse + */ + convertMetaValue(meta: JsonMetaValue): AnnotatedParse { + const source = this.sourceReconstructor.toMappedString(meta.s); + const [start, end] = this.sourceReconstructor.getOffsets(meta.s); + + switch (meta.t) { + case 'MetaString': + return { + result: typeof meta.c === 'string' ? meta.c : String(meta.c ?? ''), + kind: 'MetaString', + source, + components: [], + start, + end + }; + + case 'MetaBool': + return { + result: typeof meta.c === 'boolean' ? meta.c : Boolean(meta.c), + kind: 'MetaBool', + source, + components: [], + start, + end + }; + + case 'MetaInlines': + return { + result: meta.c as JSONValue, // Array of inline JSON objects AS-IS + kind: this.extractKind(meta), // Handle tagged values + source, + components: [], // Empty - cannot track internal locations yet + start, + end + }; + + case 'MetaBlocks': + return { + result: meta.c as JSONValue, // Array of block JSON objects AS-IS + kind: 'MetaBlocks', + source, + components: [], + start, + end + }; + + case 'MetaList': + return this.convertMetaList(meta, source, start, end); + + case 'MetaMap': + return this.convertMetaMap(meta, source, start, end); + + default: + // Unknown type - return as-is with generic kind + return { + result: meta.c as JSONValue, + kind: meta.t, + source, + components: [], + start, + end + }; + } + } + + /** + * Convert MetaList to AnnotatedParse + */ + private convertMetaList( + meta: JsonMetaValue, + source: ReturnType, + start: number, + end: number + ): AnnotatedParse { + // Runtime type check + if (!isMetaValueArray(meta.c)) { + // Return empty list if content is not an array + return { + result: [], + kind: 'MetaList', + source, + components: [], + start, + end + }; + } + + const items = meta.c.map(item => this.convertMetaValue(item)); + + return { + result: items.map(item => item.result), + kind: 'MetaList', + source, + components: items, + start, + end + }; + } + + /** + * Convert MetaMap to AnnotatedParse with interleaved key/value components + */ + private convertMetaMap( + meta: JsonMetaValue, + source: ReturnType, + start: number, + end: number + ): AnnotatedParse { + // Runtime type check + if (!isMetaMapContent(meta.c)) { + // Return empty map if content is not valid + return { + result: {}, + kind: 'MetaMap', + source, + components: [], + start, + end + }; + } + + const entries = meta.c.entries; + const components: AnnotatedParse[] = []; + const result: Record = {}; + + for (const entry of entries) { + const keySource = this.sourceReconstructor.toMappedString(entry.key_source); + const [keyStart, keyEnd] = this.sourceReconstructor.getOffsets(entry.key_source); + + const keyAP: AnnotatedParse = { + result: entry.key, + kind: 'key', + source: keySource, + components: [], + start: keyStart, + end: keyEnd + }; + + const valueAP = this.convertMetaValue(entry.value); + + // Interleave key and value in components (matches js-yaml pattern) + components.push(keyAP, valueAP); + result[entry.key] = valueAP.result; + } + + return { + result, + kind: 'MetaMap', + source, + components, + start, + end + }; + } + + /** + * Extract kind with special tag handling for YAML tagged values + * + * TODO: For now, use simple encoding like "MetaInlines:tagged:expr" + * Future enhancement: Modify @quarto/mapped-string to add optional tag field + * to AnnotatedParse interface, then use that instead + */ + private extractKind(meta: JsonMetaValue): string { + if (meta.t !== 'MetaInlines' || !Array.isArray(meta.c) || meta.c.length === 0) { + return meta.t; + } + + // Check if wrapped in Span with yaml-tagged-string class + const first = meta.c[0]; + if (!isTaggedSpan(first)) { + return 'MetaInlines'; + } + + const [attrs, _content] = first.c; + + // Check if attrs.c is an array containing 'yaml-tagged-string' + if (!Array.isArray(attrs.c) || !attrs.c.includes('yaml-tagged-string')) { + return 'MetaInlines'; + } + + // Find the tag in kv pairs + if (attrs.kv) { + const tagPair = attrs.kv.find(([k, _]) => k === 'tag'); + if (tagPair) { + const tag = tagPair[1]; + return `MetaInlines:tagged:${tag}`; + } + } + + return 'MetaInlines'; + } +} diff --git a/ts-packages/rust-qmd-json/src/source-map.ts b/ts-packages/rust-qmd-json/src/source-map.ts new file mode 100644 index 0000000..7cb1eb3 --- /dev/null +++ b/ts-packages/rust-qmd-json/src/source-map.ts @@ -0,0 +1,305 @@ +/** + * SourceInfo Reconstruction + * + * Converts pooled SourceInfo from quarto-markdown-pandoc JSON output + * into MappedString objects from @quarto/mapped-string. + */ + +import { MappedString, asMappedString, mappedConcat, mappedSubstring } from '@quarto/mapped-string'; + +/** + * Serialized SourceInfo from the JSON pool + */ +export interface SerializableSourceInfo { + r: [number, number]; // [start_offset, end_offset] + t: number; // type code (0=Original, 1=Substring, 2=Concat) + d: unknown; // type-specific data (varies by type) +} + +/** + * Type guard for Concat data structure + */ +function isConcatData(data: unknown): data is { pieces: [number, number, number][] } { + return ( + typeof data === 'object' && + data !== null && + 'pieces' in data && + Array.isArray((data as { pieces: unknown }).pieces) + ); +} + +/** + * Source context containing file information + */ +export interface SourceContext { + files: Array<{ + id: number; + path: string; + content: string; + }>; +} + +/** + * Resolved SourceInfo pointing to a file location + */ +interface ResolvedSource { + file_id: number; + range: [number, number]; +} + +/** + * Error handler callback for SourceInfo reconstruction errors + */ +export type SourceInfoErrorHandler = (msg: string, id?: number) => void; + +/** + * Default error handler that throws on errors + */ +const defaultErrorHandler: SourceInfoErrorHandler = (msg: string, id?: number) => { + const idStr = id !== undefined ? ` (SourceInfo ID: ${id})` : ''; + throw new Error(`SourceInfo reconstruction error: ${msg}${idStr}`); +}; + +/** + * Reconstructs SourceInfo from pooled format to MappedString objects + */ +export class SourceInfoReconstructor { + private pool: SerializableSourceInfo[]; + private sourceContext: SourceContext; + private errorHandler: SourceInfoErrorHandler; + private resolvedCache = new Map(); + private mappedStringCache = new Map(); + + constructor( + pool: SerializableSourceInfo[], + sourceContext: SourceContext, + errorHandler?: SourceInfoErrorHandler + ) { + this.pool = pool; + this.sourceContext = sourceContext; + this.errorHandler = errorHandler || defaultErrorHandler; + } + + /** + * Convert SourceInfo ID to MappedString + */ + toMappedString(id: number): MappedString { + // Check cache first + const cached = this.mappedStringCache.get(id); + if (cached) { + return cached; + } + + // Validate ID + if (id < 0 || id >= this.pool.length) { + this.errorHandler(`Invalid SourceInfo ID ${id} (pool size: ${this.pool.length})`, id); + // Return empty MappedString as fallback + return asMappedString(''); + } + + const info = this.pool[id]; + let result: MappedString; + + switch (info.t) { + case 0: // Original + result = this.handleOriginal(id, info); + break; + case 1: // Substring + result = this.handleSubstring(id, info); + break; + case 2: // Concat + result = this.handleConcat(id, info); + break; + default: + this.errorHandler(`Unknown SourceInfo type ${info.t}`, id); + result = asMappedString(''); + } + + // Cache and return + this.mappedStringCache.set(id, result); + return result; + } + + /** + * Get offsets from SourceInfo (without creating full MappedString) + */ + getOffsets(id: number): [number, number] { + if (id < 0 || id >= this.pool.length) { + this.errorHandler(`Invalid SourceInfo ID ${id}`, id); + return [0, 0]; + } + return this.pool[id].r; + } + + /** + * Handle Original SourceInfo type (t=0) + * Data format: file_id (number) + */ + private handleOriginal(id: number, info: SerializableSourceInfo): MappedString { + // Runtime type check + if (typeof info.d !== 'number') { + this.errorHandler(`Original SourceInfo data must be a number (file_id), got ${typeof info.d}`, id); + return asMappedString(''); + } + + const fileId = info.d; + const [start, end] = info.r; + + // Find file in context + const file = this.sourceContext.files.find(f => f.id === fileId); + if (!file) { + this.errorHandler(`File ID ${fileId} not found in source context`, id); + return asMappedString(''); + } + + // Extract substring from file content + const content = file.content.substring(start, end); + return asMappedString(content, file.path); + } + + /** + * Handle Substring SourceInfo type (t=1) + * Data format: parent_id (number) + * The range in info.r is relative to the parent's content + */ + private handleSubstring(id: number, info: SerializableSourceInfo): MappedString { + // Runtime type check + if (typeof info.d !== 'number') { + this.errorHandler(`Substring SourceInfo data must be a number (parent_id), got ${typeof info.d}`, id); + return asMappedString(''); + } + + const parentId = info.d; + const [localStart, localEnd] = info.r; + + // Get parent MappedString (recursive, with caching) + const parent = this.toMappedString(parentId); + + // Create substring with offset mapping + return mappedSubstring(parent, localStart, localEnd); + } + + /** + * Handle Concat SourceInfo type (t=2) + * Data format: {pieces: [[source_info_id, offset, length], ...]} + */ + private handleConcat(id: number, info: SerializableSourceInfo): MappedString { + // Runtime type check + if (!isConcatData(info.d)) { + this.errorHandler(`Invalid Concat data format (expected {pieces: [...]}), got ${typeof info.d}`, id); + return asMappedString(''); + } + + const pieces = info.d.pieces; + + // Build MappedString array from pieces + const mappedPieces: MappedString[] = []; + for (const [pieceId, offset, length] of pieces) { + const pieceMapped = this.toMappedString(pieceId); + // Extract substring at specified offset/length + const substring = mappedSubstring(pieceMapped, offset, offset + length); + mappedPieces.push(substring); + } + + // Concatenate all pieces + if (mappedPieces.length === 0) { + return asMappedString(''); + } + if (mappedPieces.length === 1) { + return mappedPieces[0]; + } + + return mappedConcat(mappedPieces); + } + + /** + * Recursively resolve SourceInfo chains to find original file location + * This is cached to avoid re-resolving deep chains + */ + private resolveChain(id: number): ResolvedSource { + // Check cache first + const cached = this.resolvedCache.get(id); + if (cached) { + return cached; + } + + // Validate ID + if (id < 0 || id >= this.pool.length) { + this.errorHandler(`Invalid SourceInfo ID ${id}`, id); + return { file_id: -1, range: [0, 0] }; + } + + const info = this.pool[id]; + let resolved: ResolvedSource; + + switch (info.t) { + case 0: // Original - base case + { + if (typeof info.d !== 'number') { + this.errorHandler(`Original SourceInfo data must be a number`, id); + resolved = { file_id: -1, range: info.r }; + } else { + resolved = { + file_id: info.d, + range: info.r + }; + } + } + break; + + case 1: // Substring - chain through parent + { + if (typeof info.d !== 'number') { + this.errorHandler(`Substring SourceInfo data must be a number`, id); + resolved = { file_id: -1, range: info.r }; + } else { + const parentResolved = this.resolveChain(info.d); + const [localStart, localEnd] = info.r; + const [parentStart, _] = parentResolved.range; + resolved = { + file_id: parentResolved.file_id, + range: [parentStart + localStart, parentStart + localEnd] + }; + } + } + break; + + case 2: // Concat - use first piece's resolution + // TODO: Concat doesn't have a single file location, so we use the first piece + // For error reporting, this may not be ideal + { + if (!isConcatData(info.d)) { + this.errorHandler(`Invalid Concat data format`, id); + resolved = { file_id: -1, range: info.r }; + } else { + const pieces = info.d.pieces; + if (pieces.length === 0) { + this.errorHandler(`Empty Concat pieces`, id); + resolved = { file_id: -1, range: info.r }; + } else { + const [firstPieceId, offset, length] = pieces[0]; + const firstResolved = this.resolveChain(firstPieceId); + // Offset into the first piece + const [pieceStart, _] = firstResolved.range; + resolved = { + file_id: firstResolved.file_id, + range: [pieceStart + offset, pieceStart + offset + length] + }; + } + } + } + break; + + default: + this.errorHandler(`Unknown SourceInfo type ${info.t}`, id); + resolved = { file_id: -1, range: [0, 0] }; + } + + // Cache and return + this.resolvedCache.set(id, resolved); + return resolved; + } + + // TODO: Implement circular reference detection + // This would require tracking visited IDs during resolveChain traversal +} diff --git a/ts-packages/rust-qmd-json/src/types.ts b/ts-packages/rust-qmd-json/src/types.ts new file mode 100644 index 0000000..a1408f3 --- /dev/null +++ b/ts-packages/rust-qmd-json/src/types.ts @@ -0,0 +1,64 @@ +/** + * Type definitions for quarto-markdown-pandoc JSON format + * and AnnotatedParse structures + */ + +import type { MappedString } from '@quarto/mapped-string'; +import type { SerializableSourceInfo } from './source-map.js'; + +/** + * JSON value type (matching quarto-cli's JSONValue) + */ +export type JSONValue = + | string + | number + | boolean + | null + | JSONValue[] + | { [key: string]: JSONValue }; + +/** + * AnnotatedParse structure (matching quarto-cli's interface) + */ +export interface AnnotatedParse { + start: number; + end: number; + result: JSONValue; + kind: string; + source: MappedString; + components: AnnotatedParse[]; +} + +/** + * MetaValue from quarto-markdown-pandoc JSON + */ +export interface JsonMetaValue { + t: string; // Type: "MetaString", "MetaBool", "MetaInlines", "MetaBlocks", "MetaList", "MetaMap" + c?: unknown; // Content (varies by type) + s: number; // SourceInfo ID +} + +/** + * MetaMap entry structure + */ +export interface MetaMapEntry { + key: string; + key_source: number; // SourceInfo ID for key + value: JsonMetaValue; +} + +/** + * Complete JSON output from quarto-markdown-pandoc + */ +export interface RustQmdJson { + meta: Record; + blocks: unknown[]; // Not used in metadata conversion + source_pool: SerializableSourceInfo[]; + source_context: { + files: Array<{ + id: number; + path: string; + content: string; + }>; + }; +} diff --git a/ts-packages/rust-qmd-json/test/basic.test.ts b/ts-packages/rust-qmd-json/test/basic.test.ts new file mode 100644 index 0000000..542ab7f --- /dev/null +++ b/ts-packages/rust-qmd-json/test/basic.test.ts @@ -0,0 +1,62 @@ +/** + * Basic test to verify @quarto/mapped-string integration + */ + +import { test } from 'node:test'; +import assert from 'node:assert'; +import { MappedString, asMappedString, mappedSubstring } from '@quarto/mapped-string'; + +test('can import and use @quarto/mapped-string', () => { + const str = asMappedString("Hello, World!", "test.txt"); + + assert.strictEqual(str.value, "Hello, World!"); + assert.strictEqual(str.fileName, "test.txt"); + + // Test that map function works + const result = str.map(0); + assert.ok(result !== undefined); + assert.strictEqual(result.index, 0); + assert.strictEqual(result.originalString, str); +}); + +test('can create mapped substrings', () => { + const original = asMappedString("Hello, World!", "test.txt"); + const sub = mappedSubstring(original, 7, 12); + + assert.strictEqual(sub.value, "World"); + assert.strictEqual(sub.fileName, "test.txt"); + + // Map through the substring + const result = sub.map(0); + assert.ok(result !== undefined); + assert.strictEqual(result.index, 7); // Should map to offset 7 in original +}); + +test('can convert complete JSON to AnnotatedParse', async () => { + const { parseRustQmdMetadata } = await import('../src/index.js'); + + const json = { + meta: { + title: { t: 'MetaString', c: 'Hello World', s: 0 }, + author: { t: 'MetaString', c: 'Alice', s: 1 } + }, + blocks: [], + source_pool: [ + { r: [11, 22], t: 0, d: 0 }, // "Hello World" + { r: [31, 36], t: 0, d: 0 } // "Alice" + ], + source_context: { + files: [ + { id: 0, path: 'test.qmd', content: '---\ntitle: Hello World\nauthor: Alice\n---' } + ] + } + }; + + const result = parseRustQmdMetadata(json); + + assert.strictEqual(result.kind, 'mapping'); + assert.strictEqual(typeof result.result, 'object'); + assert.strictEqual((result.result as any).title, 'Hello World'); + assert.strictEqual((result.result as any).author, 'Alice'); + assert.strictEqual(result.components.length, 4); // title key, title value, author key, author value +}); diff --git a/ts-packages/rust-qmd-json/test/meta-conversion.test.ts b/ts-packages/rust-qmd-json/test/meta-conversion.test.ts new file mode 100644 index 0000000..46e6849 --- /dev/null +++ b/ts-packages/rust-qmd-json/test/meta-conversion.test.ts @@ -0,0 +1,320 @@ +/** + * Tests for Metadata Conversion + */ + +import { strict as assert } from 'assert'; +import { SourceInfoReconstructor, SourceContext, SerializableSourceInfo } from '../src/source-map.js'; +import { MetadataConverter } from '../src/meta-converter.js'; +import type { JsonMetaValue } from '../src/types.js'; + +console.log('Running Metadata conversion tests...'); + +// Test 1: Simple string metadata +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\ntitle: Hello\n---' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [11, 16], t: 0, d: 0 } // "Hello" + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const converter = new MetadataConverter(reconstructor); + + const metaValue: JsonMetaValue = { + t: 'MetaString', + c: 'Hello', + s: 0 + }; + + const result = converter.convertMetaValue(metaValue); + + assert.equal(result.result, 'Hello'); + assert.equal(result.kind, 'MetaString'); + assert.equal(result.source.value, 'Hello'); + assert.equal(result.components.length, 0); + assert.equal(result.start, 11); + assert.equal(result.end, 16); + + console.log('✔ Simple string metadata works'); +} + +// Test 2: Boolean metadata +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\ntoc: true\n---' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [9, 13], t: 0, d: 0 } // "true" + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const converter = new MetadataConverter(reconstructor); + + const metaValue: JsonMetaValue = { + t: 'MetaBool', + c: true, + s: 0 + }; + + const result = converter.convertMetaValue(metaValue); + + assert.equal(result.result, true); + assert.equal(result.kind, 'MetaBool'); + assert.equal(result.components.length, 0); + + console.log('✔ Boolean metadata works'); +} + +// Test 3: Markdown in metadata (MetaInlines) +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\ntitle: My **Document**\n---' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [0, 5], t: 0, d: 0 }, // "My" + { r: [6, 7], t: 0, d: 0 }, // " " + { r: [8, 16], t: 0, d: 0 }, // "Document" + { r: [0, 30], t: 0, d: 0 } // Whole value + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const converter = new MetadataConverter(reconstructor); + + const metaValue: JsonMetaValue = { + t: 'MetaInlines', + c: [ + { t: 'Str', c: 'My', s: 0 }, + { t: 'Space', s: 1 }, + { t: 'Strong', c: [{ t: 'Str', c: 'Document', s: 2 }], s: 3 } + ], + s: 3 + }; + + const result = converter.convertMetaValue(metaValue); + + assert.equal(result.kind, 'MetaInlines'); + assert.ok(Array.isArray(result.result)); + assert.equal((result.result as any).length, 3); + assert.equal(result.components.length, 0); // Empty - cannot track internal locations yet + + console.log('✔ Markdown in metadata (MetaInlines) works'); +} + +// Test 4: MetaList +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\nauthor: [Alice, Bob]\n---' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [17, 22], t: 0, d: 0 }, // "Alice" + { r: [24, 27], t: 0, d: 0 }, // "Bob" + { r: [16, 28], t: 0, d: 0 } // Whole list + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const converter = new MetadataConverter(reconstructor); + + const metaValue: JsonMetaValue = { + t: 'MetaList', + c: [ + { t: 'MetaString', c: 'Alice', s: 0 }, + { t: 'MetaString', c: 'Bob', s: 1 } + ], + s: 2 + }; + + const result = converter.convertMetaValue(metaValue); + + assert.equal(result.kind, 'MetaList'); + assert.ok(Array.isArray(result.result)); + assert.equal((result.result as any[]).length, 2); + assert.equal((result.result as any[])[0], 'Alice'); + assert.equal((result.result as any[])[1], 'Bob'); + assert.equal(result.components.length, 2); // Non-empty - contains child AnnotatedParse + assert.equal(result.components[0].result, 'Alice'); + assert.equal(result.components[1].result, 'Bob'); + + console.log('✔ MetaList works'); +} + +// Test 5: MetaMap +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\nauthor:\n name: Alice\n email: alice@example.com\n---' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [18, 23], t: 0, d: 0 }, // "name" key + { r: [25, 30], t: 0, d: 0 }, // "Alice" + { r: [33, 38], t: 0, d: 0 }, // "email" key + { r: [40, 58], t: 0, d: 0 }, // "alice@example.com" + { r: [16, 59], t: 0, d: 0 } // Whole map + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const converter = new MetadataConverter(reconstructor); + + const metaValue: JsonMetaValue = { + t: 'MetaMap', + c: { + entries: [ + { key: 'name', key_source: 0, value: { t: 'MetaString', c: 'Alice', s: 1 } }, + { key: 'email', key_source: 2, value: { t: 'MetaString', c: 'alice@example.com', s: 3 } } + ] + }, + s: 4 + }; + + const result = converter.convertMetaValue(metaValue); + + assert.equal(result.kind, 'MetaMap'); + assert.equal(typeof result.result, 'object'); + assert.equal((result.result as any).name, 'Alice'); + assert.equal((result.result as any).email, 'alice@example.com'); + assert.equal(result.components.length, 4); // Interleaved key/value pairs + assert.equal(result.components[0].kind, 'key'); + assert.equal(result.components[0].result, 'name'); + assert.equal(result.components[1].result, 'Alice'); + assert.equal(result.components[2].kind, 'key'); + assert.equal(result.components[2].result, 'email'); + assert.equal(result.components[3].result, 'alice@example.com'); + + console.log('✔ MetaMap works'); +} + +// Test 6: Tagged YAML value (!expr) +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\ncompute: !expr x + 1\n---' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [17, 22], t: 0, d: 0 }, // "x + 1" + { r: [17, 22], t: 0, d: 0 } // Whole value + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const converter = new MetadataConverter(reconstructor); + + const metaValue: JsonMetaValue = { + t: 'MetaInlines', + c: [ + { + t: 'Span', + c: [ + { t: '', c: ['yaml-tagged-string'], kv: [['tag', 'expr']] }, + [{ t: 'Str', c: 'x + 1', s: 0 }] + ], + s: 1 + } + ], + s: 1 + }; + + const result = converter.convertMetaValue(metaValue); + + assert.equal(result.kind, 'MetaInlines:tagged:expr'); + assert.ok(Array.isArray(result.result)); + + console.log('✔ Tagged YAML value (!expr) works'); +} + +// Test 7: Top-level metadata conversion +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\ntitle: Hello\nauthor: Alice\n---' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [11, 16], t: 0, d: 0 }, // "Hello" + { r: [25, 30], t: 0, d: 0 } // "Alice" + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const converter = new MetadataConverter(reconstructor); + + const jsonMeta: Record = { + title: { t: 'MetaString', c: 'Hello', s: 0 }, + author: { t: 'MetaString', c: 'Alice', s: 1 } + }; + + const result = converter.convertMeta(jsonMeta); + + assert.equal(result.kind, 'mapping'); + assert.equal(typeof result.result, 'object'); + assert.equal((result.result as any).title, 'Hello'); + assert.equal((result.result as any).author, 'Alice'); + assert.equal(result.components.length, 4); // Interleaved: title key, title value, author key, author value + + console.log('✔ Top-level metadata conversion works'); +} + +// Test 8: Nested structures +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\nauthor:\n - name: Alice\n email: alice@example.com\n---' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [20, 25], t: 0, d: 0 }, // "name" key + { r: [27, 32], t: 0, d: 0 }, // "Alice" + { r: [37, 42], t: 0, d: 0 }, // "email" key + { r: [44, 62], t: 0, d: 0 }, // "alice@example.com" + { r: [18, 63], t: 0, d: 0 }, // Whole map + { r: [16, 64], t: 0, d: 0 } // Whole list + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const converter = new MetadataConverter(reconstructor); + + const metaValue: JsonMetaValue = { + t: 'MetaList', + c: [ + { + t: 'MetaMap', + c: { + entries: [ + { key: 'name', key_source: 0, value: { t: 'MetaString', c: 'Alice', s: 1 } }, + { key: 'email', key_source: 2, value: { t: 'MetaString', c: 'alice@example.com', s: 3 } } + ] + }, + s: 4 + } + ], + s: 5 + }; + + const result = converter.convertMetaValue(metaValue); + + assert.equal(result.kind, 'MetaList'); + assert.ok(Array.isArray(result.result)); + assert.equal((result.result as any[]).length, 1); + assert.equal(typeof (result.result as any[])[0], 'object'); + assert.equal((result.result as any[])[0].name, 'Alice'); + assert.equal((result.result as any[])[0].email, 'alice@example.com'); + + console.log('✔ Nested structures work'); +} + +console.log('\nAll Metadata conversion tests passed! ✨'); diff --git a/ts-packages/rust-qmd-json/test/source-map.test.ts b/ts-packages/rust-qmd-json/test/source-map.test.ts new file mode 100644 index 0000000..66de198 --- /dev/null +++ b/ts-packages/rust-qmd-json/test/source-map.test.ts @@ -0,0 +1,229 @@ +/** + * Tests for SourceInfo reconstruction + */ + +import { strict as assert } from 'assert'; +import { SourceInfoReconstructor, SerializableSourceInfo, SourceContext } from '../src/source-map.js'; + +console.log('Running SourceInfo reconstruction tests...'); + +// Test 1: Original SourceInfo type +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: 'Hello World' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [0, 5], t: 0, d: 0 } // "Hello" + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const mapped = reconstructor.toMappedString(0); + + assert.equal(mapped.value, 'Hello'); + assert.equal(mapped.fileName, 'test.qmd'); + console.log('✔ Original SourceInfo type works'); +} + +// Test 2: Substring SourceInfo type +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: 'Hello World' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [0, 11], t: 0, d: 0 }, // "Hello World" (Original) + { r: [6, 11], t: 1, d: 0 } // "World" (Substring of Original) + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const mapped = reconstructor.toMappedString(1); + + assert.equal(mapped.value, 'World'); + assert.equal(mapped.fileName, 'test.qmd'); + console.log('✔ Substring SourceInfo type works'); +} + +// Test 3: Nested Substring (Substring of Substring) +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: 'Hello World' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [0, 11], t: 0, d: 0 }, // "Hello World" (Original) + { r: [6, 11], t: 1, d: 0 }, // "World" (Substring of Original) + { r: [0, 3], t: 1, d: 1 } // "Wor" (Substring of "World") + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const mapped = reconstructor.toMappedString(2); + + assert.equal(mapped.value, 'Wor'); + assert.equal(mapped.fileName, 'test.qmd'); + console.log('✔ Nested Substring works'); +} + +// Test 4: Concat SourceInfo type +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: 'Hello World' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [0, 5], t: 0, d: 0 }, // "Hello" (Original) + { r: [6, 11], t: 0, d: 0 }, // "World" (Original) + { + r: [0, 10], + t: 2, + d: { pieces: [[0, 0, 5], [1, 0, 5]] } // Concat "Hello" + "World" + } + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const mapped = reconstructor.toMappedString(2); + + assert.equal(mapped.value, 'HelloWorld'); + // Note: mappedConcat may not preserve fileName, so we just check that it exists + // The important part is that the value is correct + console.log('✔ Concat SourceInfo type works'); +} + +// Test 5: getOffsets method +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: 'Hello World' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [0, 5], t: 0, d: 0 }, + { r: [6, 11], t: 0, d: 0 } + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + + const [start1, end1] = reconstructor.getOffsets(0); + assert.equal(start1, 0); + assert.equal(end1, 5); + + const [start2, end2] = reconstructor.getOffsets(1); + assert.equal(start2, 6); + assert.equal(end2, 11); + + console.log('✔ getOffsets method works'); +} + +// Test 6: Error handling - invalid ID +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: 'Hello World' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [0, 5], t: 0, d: 0 } + ]; + + let errorCalled = false; + const errorHandler = (msg: string, id?: number) => { + errorCalled = true; + assert.equal(id, 999); + assert.ok(msg.includes('Invalid SourceInfo ID')); + }; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext, errorHandler); + const mapped = reconstructor.toMappedString(999); + + assert.ok(errorCalled); + assert.equal(mapped.value, ''); // Fallback to empty string + console.log('✔ Error handling for invalid ID works'); +} + +// Test 7: Error handling - missing file +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: 'Hello World' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [0, 5], t: 0, d: 999 } // File ID 999 doesn't exist + ]; + + let errorCalled = false; + const errorHandler = (msg: string, id?: number) => { + errorCalled = true; + assert.ok(msg.includes('File ID 999 not found')); + }; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext, errorHandler); + const mapped = reconstructor.toMappedString(0); + + assert.ok(errorCalled); + assert.equal(mapped.value, ''); // Fallback to empty string + console.log('✔ Error handling for missing file works'); +} + +// Test 8: Caching behavior +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: 'Hello World' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [0, 5], t: 0, d: 0 } + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + + const mapped1 = reconstructor.toMappedString(0); + const mapped2 = reconstructor.toMappedString(0); + + // Should return the same cached object + assert.equal(mapped1, mapped2); + console.log('✔ Caching works correctly'); +} + +// Test 9: MappedString source mapping +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\ntitle: My **Document**\n---' } + ] + }; + + // Simulate "My **Document**" at offset 11-26 + const pool: SerializableSourceInfo[] = [ + { r: [11, 26], t: 0, d: 0 } + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const mapped = reconstructor.toMappedString(0); + + assert.equal(mapped.value, 'My **Document**'); + + // Test that map function exists and can be called + // Note: The actual line/column mapping depends on how asMappedString + // creates the mapping, which may need additional context + const mapResult = mapped.map(0); + // Just verify that mapping exists and returns something + // The actual line/column values depend on the MappedString implementation + + console.log('✔ MappedString source mapping works'); +} + +console.log('\nAll SourceInfo reconstruction tests passed! ✨'); diff --git a/ts-packages/rust-qmd-json/test/type-safety.test.ts b/ts-packages/rust-qmd-json/test/type-safety.test.ts new file mode 100644 index 0000000..e5cb68d --- /dev/null +++ b/ts-packages/rust-qmd-json/test/type-safety.test.ts @@ -0,0 +1,206 @@ +/** + * Tests for runtime type safety + */ + +import { strict as assert } from 'assert'; +import { SourceInfoReconstructor, SourceContext, SerializableSourceInfo } from '../src/source-map.js'; +import { MetadataConverter } from '../src/meta-converter.js'; +import type { JsonMetaValue } from '../src/types.js'; + +console.log('Running type safety tests...'); + +// Test 1: Invalid Original SourceInfo data type +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: 'Hello World' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [0, 5], t: 0, d: 'invalid' as unknown } // Should be number + ]; + + let errorCalled = false; + const errorHandler = (msg: string) => { + errorCalled = true; + assert.ok(msg.includes('must be a number')); + }; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext, errorHandler); + const mapped = reconstructor.toMappedString(0); + + assert.ok(errorCalled); + assert.equal(mapped.value, ''); // Fallback + console.log('✔ Invalid Original data type caught'); +} + +// Test 2: Invalid Substring SourceInfo data type +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: 'Hello World' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [0, 11], t: 0, d: 0 }, + { r: [6, 11], t: 1, d: { invalid: 'object' } as unknown } // Should be number + ]; + + let errorCalled = false; + const errorHandler = (msg: string) => { + errorCalled = true; + assert.ok(msg.includes('must be a number')); + }; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext, errorHandler); + const mapped = reconstructor.toMappedString(1); + + assert.ok(errorCalled); + assert.equal(mapped.value, ''); // Fallback + console.log('✔ Invalid Substring data type caught'); +} + +// Test 3: Invalid Concat SourceInfo data type +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: 'Hello World' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [0, 5], t: 0, d: 0 }, + { r: [0, 10], t: 2, d: 'not-an-object' as unknown } // Should be {pieces: [...]} + ]; + + let errorCalled = false; + const errorHandler = (msg: string) => { + errorCalled = true; + assert.ok(msg.includes('Invalid Concat data format')); + }; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext, errorHandler); + const mapped = reconstructor.toMappedString(1); + + assert.ok(errorCalled); + assert.equal(mapped.value, ''); // Fallback + console.log('✔ Invalid Concat data type caught'); +} + +// Test 4: Invalid MetaList content type +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\nauthor: invalid\n---' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [16, 23], t: 0, d: 0 } + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const converter = new MetadataConverter(reconstructor); + + const metaValue: JsonMetaValue = { + t: 'MetaList', + c: 'not-an-array' as unknown, // Should be array + s: 0 + }; + + const result = converter.convertMetaValue(metaValue); + + assert.equal(result.kind, 'MetaList'); + assert.deepEqual(result.result, []); // Fallback to empty array + assert.equal(result.components.length, 0); + console.log('✔ Invalid MetaList content type handled gracefully'); +} + +// Test 5: Invalid MetaMap content type +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\nauthor: invalid\n---' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [16, 23], t: 0, d: 0 } + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const converter = new MetadataConverter(reconstructor); + + const metaValue: JsonMetaValue = { + t: 'MetaMap', + c: 'not-an-object' as unknown, // Should be {entries: [...]} + s: 0 + }; + + const result = converter.convertMetaValue(metaValue); + + assert.equal(result.kind, 'MetaMap'); + assert.deepEqual(result.result, {}); // Fallback to empty object + assert.equal(result.components.length, 0); + console.log('✔ Invalid MetaMap content type handled gracefully'); +} + +// Test 6: Non-string MetaString content +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\ntitle: 123\n---' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [11, 14], t: 0, d: 0 } + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const converter = new MetadataConverter(reconstructor); + + const metaValue: JsonMetaValue = { + t: 'MetaString', + c: 123 as unknown, // Number instead of string + s: 0 + }; + + const result = converter.convertMetaValue(metaValue); + + assert.equal(result.kind, 'MetaString'); + assert.equal(result.result, '123'); // Converted to string + console.log('✔ Non-string MetaString content converted'); +} + +// Test 7: Non-boolean MetaBool content +{ + const sourceContext: SourceContext = { + files: [ + { id: 0, path: 'test.qmd', content: '---\ntoc: 1\n---' } + ] + }; + + const pool: SerializableSourceInfo[] = [ + { r: [9, 10], t: 0, d: 0 } + ]; + + const reconstructor = new SourceInfoReconstructor(pool, sourceContext); + const converter = new MetadataConverter(reconstructor); + + const metaValue: JsonMetaValue = { + t: 'MetaBool', + c: 1 as unknown, // Number instead of boolean + s: 0 + }; + + const result = converter.convertMetaValue(metaValue); + + assert.equal(result.kind, 'MetaBool'); + assert.equal(result.result, true); // Converted to boolean + console.log('✔ Non-boolean MetaBool content converted'); +} + +console.log('\nAll type safety tests passed! ✨'); diff --git a/ts-packages/rust-qmd-json/tsconfig.json b/ts-packages/rust-qmd-json/tsconfig.json new file mode 100644 index 0000000..4553cf1 --- /dev/null +++ b/ts-packages/rust-qmd-json/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "lib": ["ES2022"], + "moduleResolution": "node", + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "resolveJsonModule": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "test"] +}