fix: Add convertToBtrie method to trie-lib (#8562)

Signed-off-by: Jason Dent <Jason3S@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: Jason3S

packages/cspell-dictionary-bundler-plugin/src/core/bunder.test.ts

@@ -1,9 +1,11 @@
 import fs from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
 
 import type { CSpellVFS } from '@cspell/cspell-types';
+import type { ICSpellConfigFile } from 'cspell-config-lib';
 import { describe, expect, test } from 'vitest';
 
-import { makeVfsUrl, populateVfs } from './bundler.ts';
+import { makeVfsUrl, populateVfs, resolveDictionaries } from './bundler.ts';
 
 const fixturesUrl = new URL('../../tests/fixtures/file.txt', import.meta.url);
 
@@ -25,11 +27,43 @@ describe('populateVfs', () => {
     test('should populate the vfs with the content of the file', async () => {
         const vfs: CSpellVFS = {};
         const fileUrl = new URL('words.txt', fixturesUrl);
-        const url = await populateVfs(vfs, fileUrl);
+        const url = await populateVfs(vfs, { url: fileUrl });
         expect(url.href).toMatch(/^cspell-vfs:\/\/\//);
         expect(vfs[url.href]).toBeDefined();
         expect(vfs[url.href].encoding).toBe('base64');
         const content = Buffer.from(vfs[url.href].data as string, 'base64').toString();
         expect(content).toBe(await fs.readFile(fileUrl, 'utf8'));
     });
 });
+
+describe('resolveDictionaries', () => {
+    test('should resolve dictionary definitions and populate the vfs', async () => {
+        const url = new URL('test/config.json', import.meta.url);
+        const config: ICSpellConfigFile = {
+            url,
+            settings: {
+                dictionaryDefinitions: [{ name: 'test-dict', path: fileURLToPath(new URL('words.txt', fixturesUrl)) }],
+            },
+        };
+
+        const settings = await resolveDictionaries(config, {
+            convertToBTrie: true,
+            minConvertSize: 0,
+            compress: true,
+            debug: false,
+        });
+
+        expect(settings.dictionaryDefinitions).toBeDefined();
+        expect(settings.dictionaryDefinitions?.[0].name).toBe('test-dict');
+        expect(settings.dictionaryDefinitions?.[0].file).toBeUndefined();
+        expect(settings.dictionaryDefinitions?.[0].btrie).toBeUndefined();
+        expect(settings.dictionaryDefinitions?.[0].path).toEqual(expect.stringContaining('cspell-vfs:///'));
+
+        const vfsPath = settings.dictionaryDefinitions?.[0].path as string;
+        const vfsUrl = new URL(vfsPath);
+        expect(vfsUrl.protocol).toBe('cspell-vfs:');
+        expect(vfsUrl.pathname).toEqual(expect.stringMatching(/\.btrie.gz$/));
+
+        expect(settings.vfs?.[vfsPath]).toBeDefined();
+    });
+});

packages/cspell-dictionary-bundler-plugin/src/core/bundler.ts

@@ -1,15 +1,20 @@
 import { createHash } from 'node:crypto';
 import fs from 'node:fs/promises';
 import { createRequire } from 'node:module';
-import { pathToFileURL } from 'node:url';
+import nodePath from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { gzipSync } from 'node:zlib';
 
 import type { CSpellSettings, CSpellVFS } from '@cspell/cspell-types';
 import { mergeConfig } from '@cspell/cspell-types';
 import type { CSpellConfigFile, CSpellConfigFileReaderWriter, ICSpellConfigFile } from 'cspell-config-lib';
+import { convertToBTrie } from 'cspell-trie-lib';
 
-export interface CSpellDictionaryBundlerOptions {
-    debug?: boolean;
-}
+import type { Options } from './options.ts';
+
+export type CSpellDictionaryBundlerOptions = Required<
+    Pick<Options, 'debug' | 'convertToBTrie' | 'minConvertSize' | 'compress'>
+>;
 
 export class CSpellDictionaryBundler {
     #loadedConfigs = new Map<string, Promise<ICSpellConfigFile>>();
@@ -43,7 +48,7 @@ export class CSpellDictionaryBundler {
         const imports = await this.loadImports(config);
         const settings = mergeConfig(
             imports.map((f) => f.settings),
-            await this.resolveDictionaries(config),
+            await resolveDictionaries(config, this.#options),
         );
         delete settings.import;
         delete settings['$schema'];
@@ -53,29 +58,6 @@ export class CSpellDictionaryBundler {
         };
     }
 
-    async resolveDictionaries(config: ICSpellConfigFile): Promise<CSpellSettings> {
-        const settings = { ...config.settings };
-        if (!settings.dictionaryDefinitions) return settings;
-        // Make a copy of the dictionary definitions and vfs to avoid mutating the original config file.
-        const dictDefs = (settings.dictionaryDefinitions = [...settings.dictionaryDefinitions]);
-        const vfs: CSpellVFS = (settings.vfs ??= Object.create(null));
-
-        for (let i = 0; i < dictDefs.length; ++i) {
-            const def = dictDefs[i];
-            if (!def.path) continue;
-            const d = { ...def };
-            dictDefs[i] = d;
-            const url = new URL(def.btrie ?? def.path, config.url);
-            if (url.protocol !== 'file:') continue;
-            const vfsUrl = await populateVfs(vfs, url);
-            delete d.file;
-            delete d.btrie;
-            d.path = vfsUrl.href;
-        }
-
-        return settings;
-    }
-
     importConfig(url: URL, content?: string): Promise<CSpellConfigFile> {
         if (content && !isCodeFile(url)) {
             return Promise.resolve(this.reader.parse({ url, content }));
@@ -89,19 +71,64 @@ export class CSpellDictionaryBundler {
     }
 }
 
+export async function resolveDictionaries(
+    config: ICSpellConfigFile,
+    options: CSpellDictionaryBundlerOptions,
+): Promise<CSpellSettings> {
+    const settings = { ...config.settings };
+    if (!settings.dictionaryDefinitions) return settings;
+    if (config.url.protocol !== 'file:') return settings;
+    // Make a copy of the dictionary definitions and vfs to avoid mutating the original config file.
+    const dictDefs = (settings.dictionaryDefinitions = [...settings.dictionaryDefinitions]);
+    const vfs: CSpellVFS = (settings.vfs ??= Object.create(null));
+    const minConvertSize = options.minConvertSize ?? 1024;
+
+    for (let i = 0; i < dictDefs.length; ++i) {
+        const def = dictDefs[i];
+        const d = { ...def };
+        if (!d.path) continue;
+        dictDefs[i] = d;
+        const url = resolvePath(d.btrie ?? d.path, config.url);
+        if (url.protocol !== 'file:') continue;
+        let file = await readFile({ url });
+        file = options.convertToBTrie && fileLength(file) >= minConvertSize ? await convert(file) : file;
+        file = options.compress && fileLength(file) >= minConvertSize ? compressFile(file) : file;
+        const vfsUrl = await populateVfs(vfs, file);
+        delete d.file;
+        delete d.btrie;
+        d.path = vfsUrl.href;
+    }
+
+    return settings;
+}
+
+interface FileReference {
+    url: URL;
+    content?: string | Uint8Array<ArrayBuffer>;
+}
+
+interface FileResource extends FileReference {
+    content: string | Uint8Array<ArrayBuffer>;
+}
+
+async function convert(file: FileReference): Promise<FileResource> {
+    const resource = await readFile(file);
+    return convertToBTrie(resource, { optimize: true });
+}
+
 /**
  * Load a file from the file system and populate the virtual file system with its content.
  *
  * @param vfs - The Virtual Files system data
  * @param url - The url to load and store.
  * @return The cspell-vfs url that was loaded.
  */
-export async function populateVfs(vfs: CSpellVFS, url: URL): Promise<URL> {
-    const content = await fs.readFile(url);
+export async function populateVfs(vfs: CSpellVFS, fileRef: FileReference): Promise<URL> {
+    const { url, content } = await readFile(fileRef);
 
     const hash = createHash('sha256').update(content).digest('hex');
 
-    const data = content.toString('base64');
+    const data = typeof content === 'string' ? content : Buffer.from(content).toString('base64');
     const vfsUrl = makeVfsUrl(url, hash.slice(0, 16));
     vfs[vfsUrl.href] = {
         data,
@@ -150,3 +177,38 @@ const isCodeFileRegExp = /\.[cm]?(js|ts)$/i;
 function isCodeFile(url: URL): boolean {
     return isCodeFileRegExp.test(url.pathname);
 }
+
+async function readFile(fileRef: FileReference): Promise<FileResource> {
+    const url = fileRef.url;
+    const content = fileRef.content ?? (await fs.readFile(url));
+    return { url, content };
+}
+
+/**
+ * This is the approximate size of the file in bytes.
+ * @param file
+ * @returns
+ */
+function fileLength(file: FileResource): number {
+    if (typeof file.content === 'string') {
+        return file.content.length;
+    }
+    return file.content.byteLength;
+}
+
+function compressFile(file: FileResource): FileResource {
+    if (file.url.pathname.endsWith('.gz')) return file;
+    const url = new URL(file.url.pathname + '.gz', file.url);
+    const content = gzipSync(file.content);
+    return { url, content };
+}
+
+function resolvePath(path: string, base: URL): URL {
+    if (isUrlLike(path)) {
+        return new URL(path);
+    }
+
+    const dir = fileURLToPath(new URL('./', base));
+    const filePath = nodePath.resolve(dir, path);
+    return pathToFileURL(filePath);
+}

packages/cspell-dictionary-bundler-plugin/src/core/core.ts

@@ -99,6 +99,9 @@ export function resolveOptions(options: Options): OptionsResolved {
         include: options.include || [/.*cspell(?:[-]ext)?(\..*)?\.(?:jsonc?|ya?ml|toml)$/i],
         exclude: options.exclude || undefined,
         enforce: 'enforce' in options ? options.enforce : 'pre',
+        convertToBTrie: options.convertToBTrie ?? true,
+        minConvertSize: options.minConvertSize ?? 200,
+        compress: options.compress ?? false,
         debug: !!options.debug,
     };
 }

packages/cspell-dictionary-bundler-plugin/src/core/options.ts

@@ -4,5 +4,22 @@ export interface Options {
     include?: FilterPattern | undefined;
     exclude?: FilterPattern | undefined;
     enforce?: 'pre' | 'post' | undefined;
+    /**
+     * Convert the dictionary to a BTrie format.
+     * This will increase the size of the output file, but will significantly reduce the time it takes to load the dictionary.
+     * @default true
+     */
+    convertToBTrie?: boolean | undefined;
+    /**
+     * The minimum size (in bytes) a dictionary must be to be converted to BTrie format.
+     * @default 200
+     */
+    minConvertSize?: number | undefined;
+
+    /**
+     * Compress the inline data using gzip.
+     */
+    compress?: boolean | undefined;
+
     debug?: boolean;
 }

packages/cspell-dictionary-bundler-plugin/tests/__snapshots__/rollup.test.ts.snap

@@ -16,17 +16,20 @@ exports[`Validate index.ts > Track changes to the API. 1`] = `
   "buildTrie",
   "buildTrieFast",
   "consolidate",
+  "convertToBTrie",
   "countNodes",
   "countWords",
   "createDictionaryLineParser",
   "createTrieRoot",
   "createTrieRootFromList",
   "createWeightedMap",
+  "decodeFile",
   "decodeTrie",
   "defaultTrieInfo",
   "defaultTrieOptions",
   "editDistance",
   "editDistanceWeighted",
+  "encodeITrieToBTrie",
   "encodeTrieDataToBTrie",
   "expandCharacterSet",
   "findNode",

packages/cspell-trie-lib/api/api.d.ts

@@ -6,4 +6,4 @@ export {
 } from './createTrieBlob.ts';
 export { TrieBlob } from './TrieBlob.ts';
 export { isBTrieData } from './TrieBlobEncoder.ts';
-export { decodeBTrie, encodeTrieDataToBTrie } from './trieDataEncoder.ts';
+export { decodeBTrie, encodeITrieToBTrie, encodeTrieDataToBTrie } from './trieDataEncoder.ts';

packages/cspell-trie-lib/src/__snapshots__/index.test.ts.snap

@@ -1,8 +1,13 @@
 import type { BuildOptions } from '../BuildOptions.ts';
+import type { ITrie } from '../ITrie.ts';
 import type { TrieData } from '../TrieData.ts';
 import { TrieBlob } from './TrieBlob.ts';
 import { TrieBlobBuilder } from './TrieBlobBuilder.ts';
 
+export function encodeITrieToBTrie(trie: ITrie, buildOptions?: BuildOptions): Uint8Array<ArrayBuffer> {
+    return encodeTrieDataToBTrie(trie.data, buildOptions);
+}
+
 export function encodeTrieDataToBTrie(data: TrieData, buildOptions?: BuildOptions): Uint8Array<ArrayBuffer> {
     const needToBuild = buildOptions?.optimize || buildOptions?.useStringTable;
     if (!needToBuild && data.encodeToBTrie) {