feat: auto load btrie files if available (#8218)

Author: Jason3S

cspell-dict.txt

@@ -2,6 +2,7 @@
 alexiosc
 backreference
 bitjson
+btrie
 cfile
 cheatsheets
 codecov

package.json

@@ -44,7 +44,7 @@
     "pub-grad": "pnpm run pre-pub && lerna publish --conventional-commits --conventional-graduate --changelog-preset conventionalcommits  --sync-workspace-lock",
     "pub-next": "pnpm run pre-pub && lerna publish --conventional-commits --conventional-prerelease --changelog-preset conventionalcommits   --sync-workspace-lock --dist-tag next",
     "test-watch": "pnpm -r run --parallel test-watch",
-    "test": "pnpm -r --stream --workspace-concurrency=2 run test && pnpm run test-schema",
+    "test": "pnpm test:prep && pnpm -r --stream --workspace-concurrency=2 run test && pnpm run test-schema",
     "test:perf": "pnpm -r --stream --workspace-concurrency=2 run test:perf",
     "test:update-snapshots": "pnpm run -r test:update-snapshot",
     "test:bin": "pnpm test:bin-spell && pnpm test:bin-spell-cache-content && pnpm test:bin-spell-cache-metadata && pnpm test:bin-trace && pnpm test:bin-check && pnpm test:bin-spell-yarn",
@@ -54,6 +54,7 @@
     "test:bin-spell-cache-content": "node ./bin.mjs -c cspell.test.json --cache --cache-strategy content --cache-location temp/.cspellcache-content",
     "test:bin-spell-cache-metadata": "node ./bin.mjs -c cspell.test.json --cache --cache-strategy metadata --cache-location temp/.cspellcache-meta",
     "test:bin-trace": "node ./bin.mjs trace test",
+    "test:prep": "cspell-tools btrie node_modules/@cspell/dict-en_us/en_US.trie.gz",
     "test-build-docs": "cd website && pnpm run build:site",
     "test-integrations": "cd ./integration-tests && pnpm run integration-tests",
     "test-schema": "node ./test-packages/cspell-types/validate-schema/validate-schema.mjs",
@@ -101,6 +102,8 @@
     "node": ">=20.0.0"
   },
   "devDependencies": {
+    "@cspell/cspell-tools": "workspace:*",
+    "@cspell/dict-en_us": "^4.4.26",
     "@cspell/eslint-plugin": "workspace:*",
     "@eslint/js": "^9.39.2",
     "@internal/scripts": "workspace:*",

packages/cspell-io/package.json

@@ -23,15 +23,15 @@
     "!**/*.map"
   ],
   "scripts": {
-    "build": "tsdown",
+    "build": "tsdown && pnpm build:tsc",
+    "build:tsc": "tsc -p .",
     "build:watch": "tsdown --watch",
     "watch": "pnpm build:watch",
     "clean": "shx rm -rf dist temp coverage \"*.tsbuildInfo\"",
     "clean-build": "pnpm run clean && pnpm run build",
     "coverage": "vitest run --coverage --pool=forks",
     "test-watch": "vitest",
-    "test:build": "tsc -p .",
-    "test": "pnpm test:build && vitest run --pool=forks"
+    "test": "vitest run --pool=forks"
   },
   "repository": {
     "type": "git",

packages/cspell-io/src/CSpellIONode.ts

@@ -57,7 +57,7 @@ export class CSpellIONode implements CSpellIO {
         }
         return res.value;
     }
-    writeFile(urlOrFilename: UrlOrReference, content: string | ArrayBufferView): Promise<FileReference> {
+    writeFile(urlOrFilename: UrlOrReference, content: string | Uint8Array<ArrayBuffer>): Promise<FileReference> {
         const ref = toFileReference(urlOrFilename);
         const fileResource = CFileResource.from(ref, content);
         const res = this.serviceBus.dispatch(RequestFsWriteFile.create(fileResource));

packages/cspell-io/src/VirtualFs.test.ts

@@ -22,7 +22,7 @@ describe('VirtualFs', () => {
 
     beforeEach(() => {
         virtualFs = createVirtualFS();
-        mockConsoleLog = vi.spyOn(console, 'log');
+        mockConsoleLog = vi.spyOn(console, 'log').mockImplementation(() => {});
     });
 
     afterEach(() => {

packages/cspell-io/src/common/CFileResource.test.ts

@@ -1,9 +1,13 @@
 import { Buffer } from 'node:buffer';
+import { promisify } from 'node:util';
+import zlib from 'node:zlib';
 
 import { describe, expect, test } from 'vitest';
 
 import { CFileResource, fromFileResource } from './CFileResource.js';
 
+const gzip = promisify(zlib.gzip);
+
 describe('CFileResource', () => {
     describe('from', () => {
         test('should create a CFileResource from a FileResource', () => {
@@ -86,6 +90,32 @@ describe('CFileResource', () => {
             expect(cFileResource.baseFilename).toBe('file.txt');
             expect(cFileResource.gz).toEqual(gz);
         });
+
+        test('binary data', async () => {
+            const text = 'Hello, world!';
+            const encoder = new TextEncoder();
+            const content = encoder.encode(text);
+            const decoder = new TextDecoder();
+
+            const cFileResource = CFileResource.from(new URL('https://example.com/file.txt'), content);
+
+            const fileData = await cFileResource.getBytes();
+            expect([...fileData]).toEqual([...content]);
+            expect(decoder.decode(fileData)).toBe(text);
+        });
+
+        test('binary data compressed', async () => {
+            const text = 'Hello, world!';
+            const data = Buffer.from(text, 'utf8');
+            const content = await gzip(data);
+            const decoder = new TextDecoder();
+
+            const cFileResource = CFileResource.from(new URL('https://example.com/file.txt.gz'), content);
+
+            const fileData = await cFileResource.getBytes();
+            expect([...fileData]).toEqual([...data]);
+            expect(decoder.decode(fileData)).toBe(text);
+        });
     });
 
     describe('fromFileResource', () => {

packages/cspell-io/src/common/CFileResource.ts

@@ -1,7 +1,7 @@
 import { assert } from '../errors/assert.js';
 import type { BufferEncoding } from '../models/BufferEncoding.js';
 import type { FileReference, FileResource, TextFileResource } from '../models/FileResource.js';
-import { decode, encodeString, isGZipped } from './encode-decode.js';
+import { decode, decompress, encodeString, isGZipped } from './encode-decode.js';
 
 export interface CFileResourceJson {
     url: string;
@@ -12,41 +12,53 @@ export interface CFileResourceJson {
 }
 
 export class CFileResource implements TextFileResource {
-    private _text?: string;
     readonly baseFilename?: string | undefined;
-    private _gz?: boolean | undefined;
+    readonly url: URL;
+    readonly content: string | Uint8Array<ArrayBuffer>;
+    readonly encoding: BufferEncoding | undefined;
+    #gz?: boolean | undefined;
+    #text?: string;
+    #data?: Uint8Array<ArrayBuffer>;
 
     constructor(
-        readonly url: URL,
-        readonly content: string | ArrayBufferView,
-        readonly encoding: BufferEncoding | undefined,
+        url: URL,
+        content: string | Uint8Array<ArrayBuffer>,
+        encoding: BufferEncoding | undefined,
         baseFilename: string | undefined,
         gz: boolean | undefined,
     ) {
+        this.url = url;
+        this.content = content;
+        this.encoding = encoding;
         this.baseFilename = baseFilename ?? ((url.protocol !== 'data:' && url.pathname.split('/').pop()) || undefined);
-        this._gz = gz;
+        this.#gz = gz;
     }
 
     get gz(): boolean {
-        if (this._gz !== undefined) return this._gz;
+        if (this.#gz !== undefined) return this.#gz;
         if (this.url.pathname.endsWith('.gz')) return true;
         if (typeof this.content === 'string') return false;
         return isGZipped(this.content);
     }
 
     getText(encoding?: BufferEncoding): string {
-        if (this._text !== undefined) return this._text;
+        if (this.#text !== undefined) return this.#text;
         const text = typeof this.content === 'string' ? this.content : decode(this.content, encoding ?? this.encoding);
-        this._text = text;
+        this.#text = text;
         return text;
     }
 
-    getBytes(): Uint8Array {
-        const arrayBufferview =
-            typeof this.content === 'string' ? encodeString(this.content, this.encoding) : this.content;
-        return arrayBufferview instanceof Uint8Array
-            ? arrayBufferview
-            : new Uint8Array(arrayBufferview.buffer, arrayBufferview.byteOffset, arrayBufferview.byteLength);
+    async getBytes(unzip?: boolean): Promise<Uint8Array<ArrayBuffer>> {
+        if (unzip !== false && this.#data !== undefined) return this.#data;
+        if (typeof this.content === 'string') {
+            this.#data = encodeString(this.content, this.encoding);
+            return this.#data;
+        }
+        if (unzip ?? isGZipped(this.content)) {
+            this.#data = await decompress(this.content, 'gzip');
+            return this.#data;
+        }
+        return this.content;
     }
 
     public toJson(): CFileResourceJson {
@@ -64,18 +76,18 @@ export class CFileResource implements TextFileResource {
     }
 
     static from(fileResource: FileResource): CFileResource;
-    static from(fileReference: FileReference, content: string | ArrayBufferView): CFileResource;
-    static from(fileReference: FileReference | URL, content: string | ArrayBufferView): CFileResource;
+    static from(fileReference: FileReference, content: string | Uint8Array<ArrayBuffer>): CFileResource;
+    static from(fileReference: FileReference | URL, content: string | Uint8Array<ArrayBuffer>): CFileResource;
     static from(
         url: URL,
-        content: string | ArrayBufferView,
+        content: string | Uint8Array<ArrayBuffer>,
         encoding?: BufferEncoding | undefined,
         baseFilename?: string | undefined,
         gz?: boolean,
     ): CFileResource;
     static from(
         urlOrFileResource: FileResource | FileReference | URL,
-        content?: string | ArrayBufferView,
+        content?: string | Uint8Array<ArrayBuffer>,
         encoding?: BufferEncoding,
         baseFilename?: string | undefined,
         gz?: boolean,

packages/cspell-io/src/common/arrayBuffers.test.ts

@@ -2,21 +2,14 @@ import { Buffer } from 'node:buffer';
 
 import { describe, expect, test } from 'vitest';
 
-import {
-    arrayBufferViewToBuffer,
-    asUint8Array,
-    copyArrayBufferView,
-    sliceView,
-    swap16,
-    swap16Poly,
-} from './arrayBuffers.js';
+import { arrayBufferViewToBuffer, swap16, swap16Poly, toUint8Array } from './arrayBuffers.js';
 
 const sampleText = 'This is a bit of text to test things with.';
 
 describe('arrayBuffers', () => {
     test('asUint8Array', () => {
         const buf = Buffer.from(sampleText);
-        const u8 = asUint8Array(buf);
+        const u8 = toUint8Array(buf);
         expect(u8[0]).toBe(sampleText.codePointAt(0));
         expect(buf[0]).toBe(sampleText.codePointAt(0));
         u8[0] = 32;
@@ -44,38 +37,12 @@ describe('arrayBuffers', () => {
         expect(buf).toEqual(Buffer.from([2, 1, 4, 3]));
     });
 
-    test('sliceView', () => {
-        const src = [1, 2, 3, 4, 5, 6, 7, 8];
-        const buf = Buffer.from(src);
-        const view = sliceView(buf, 2, 4);
-        expect(view.buffer).toBe(buf.buffer);
-        const u8 = asUint8Array(view);
-        expect(u8.buffer).toBe(buf.buffer);
-        const u8From = Uint8Array.from(src);
-        const cView = copyArrayBufferView(view);
-        expect(cView.buffer).not.toBe(buf.buffer);
-        expect(cView).toEqual(Uint8Array.from([3, 4, 5, 6]));
-        expect(u8).toEqual(Uint8Array.from([3, 4, 5, 6]));
-        expect(asUint8Array(sliceView(u8From, 2, 4))).toEqual(Uint8Array.from([3, 4, 5, 6]));
-    });
-
-    test('sliceView bounds', () => {
-        const src = [1, 2, 3, 4, 5, 6, 7, 8];
-        const buf = Uint8Array.from(src);
-        const view = sliceView(buf, 2, 100);
-        expect(view.buffer).toBe(buf.buffer);
-        expect(asUint8Array(view)).toEqual(Uint8Array.from([3, 4, 5, 6, 7, 8]));
-        const view2 = sliceView(view, 2);
-        expect(view2.buffer).toBe(buf.buffer);
-        expect(asUint8Array(view2)).toEqual(Uint8Array.from([5, 6, 7, 8]));
-    });
-
     test('arrayBufferViewToBuffer', () => {
         const src = [1, 2, 3, 4, 5, 6, 7, 8];
         const buf = Buffer.from(src);
         expect(arrayBufferViewToBuffer(buf)).toBe(buf);
 
-        const view = sliceView(buf, 2, 4);
+        const view = buf.subarray(2, 6);
         const bufView = arrayBufferViewToBuffer(view);
         expect(bufView).not.toBe(buf);
         expect(bufView.buffer).toBe(buf.buffer);

packages/cspell-io/src/common/arrayBuffers.ts

@@ -1,58 +1,51 @@
 import { Buffer } from 'node:buffer';
 
+import type { TArrayBufferView } from '../types.js';
+
 /**
- * Treat a ArrayBufferView as a Uint8Array.
+ * Treat a TArrayBufferView as a Uint8Array.
  * The Uint8Array will share the same underlying ArrayBuffer.
  * @param data - source data
  * @returns Uint8Array
  */
-export function asUint8Array(data: ArrayBufferView): Uint8Array {
+export function toUint8Array(data: TArrayBufferView): Uint8Array<ArrayBuffer>;
+export function toUint8Array(data: ArrayBufferView): Uint8Array;
+export function toUint8Array(data: ArrayBufferView): Uint8Array {
+    if (data instanceof Uint8Array) {
+        return data;
+    }
     return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
 }
 
+export function arrayBufferViewToBuffer(data: TArrayBufferView): Buffer<ArrayBuffer>;
+export function arrayBufferViewToBuffer(data: ArrayBufferView): Buffer;
 export function arrayBufferViewToBuffer(data: ArrayBufferView): Buffer {
     if (data instanceof Buffer) {
         return data;
     }
-    const buf = Buffer.from(data.buffer);
-    if (data.byteOffset === 0 && data.byteLength === data.buffer.byteLength) {
-        return buf;
-    }
-    return buf.subarray(data.byteOffset, data.byteOffset + data.byteLength);
+    const buf = Buffer.from(data.buffer, data.byteOffset, data.byteLength);
+    return buf;
 }
 
 /**
  * Copy the data buffer.
  * @param data - source data
  * @returns A copy of the data
  */
-export function copyArrayBufferView(data: ArrayBufferView): ArrayBufferView {
+export function copyArrayBufferView(data: TArrayBufferView): Uint8Array<ArrayBuffer>;
+export function copyArrayBufferView(data: ArrayBufferView): Uint8Array;
+export function copyArrayBufferView(data: ArrayBufferView): Uint8Array {
     return new Uint8Array(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength));
 }
 
-/**
- * Slice an existing data view. Returns a new view using the same underlying data.
- * @param data - data view
- * @param byteOffset - offset from the beginning of the view.
- * @param byteLength - optional length
- */
-export function sliceView(data: ArrayBufferView, byteOffset: number, byteLength?: number): ArrayBufferView {
-    const currentEnd = data.byteOffset + data.byteLength;
-    const start = Math.min(data.byteOffset + byteOffset, currentEnd);
-    const end = byteLength ? Math.min(currentEnd, start + byteLength) : currentEnd;
-    return {
-        buffer: data.buffer,
-        byteOffset: start,
-        byteLength: end - start,
-    };
-}
-
 /**
  * Swap the bytes in a buffer.
  * @param data - data to swap
  * @returns data
  */
-export function swap16Poly(data: ArrayBufferView): ArrayBufferView {
+export function swap16Poly<T extends TArrayBufferView>(data: T): T;
+export function swap16Poly<T extends ArrayBufferView>(data: T): T;
+export function swap16Poly(data: TArrayBufferView): TArrayBufferView {
     const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
     for (let i = 0; i < view.byteLength; i += 2) {
         view.setUint16(i, view.getUint16(i, false), true);
@@ -65,11 +58,15 @@ export function swap16Poly(data: ArrayBufferView): ArrayBufferView {
  * @param data - data to swap
  * @returns data
  */
-export function swap16(data: ArrayBufferView): ArrayBufferView {
-    return arrayBufferViewToBuffer(data).swap16();
+export function swap16<T extends TArrayBufferView>(data: T): T;
+export function swap16<T extends ArrayBufferView>(data: T): T;
+
+export function swap16<T extends ArrayBufferView>(data: T): T {
+    arrayBufferViewToBuffer(data).swap16();
+    return data;
 }
 
-export function swapBytes(data: ArrayBufferView): ArrayBufferView {
+export function swapBytes(data: TArrayBufferView): Uint8Array<ArrayBuffer> {
     const buf = copyArrayBufferView(data);
     return swap16(buf);
 }