fix: Support loading btrie dictionaries in cspell-lib (#8311)

Jason3S · web-flow · commit 468b8ab4e354 · 2026-01-07T11:57:56.000+01:00
diff --git a/packages/cspell-lib/fixtures/dictionaries/btrie/README.md b/packages/cspell-lib/fixtures/dictionaries/btrie/README.md
@@ -0,0 +1,11 @@
+# BTrie Dictionaries
+
+This is an example config file with both a path to a text file and a path to a BTrie file.
+
+The btrie can be build using:
+
+```
+cspell-tools btrie ./btrie-words.txt
+```
+
+It contains some words not found in `words.txt`. Example: `Exxtra`.
diff --git a/packages/cspell-lib/fixtures/dictionaries/btrie/btrie-words.btrie.gz b/packages/cspell-lib/fixtures/dictionaries/btrie/btrie-words.btrie.gz
diff --git a/packages/cspell-lib/fixtures/dictionaries/btrie/btrie-words.txt b/packages/cspell-lib/fixtures/dictionaries/btrie/btrie-words.txt
@@ -0,0 +1,29 @@
+Amsterdam
+Angles
+cities
+City
+Configuration
+Default
+define
+Definitions
+Delhi
+dictionaries
+dictionary
+false
+file
+Francisco
+fully
+Inline
+Las
+load
+London
+Mexico
+name
+New
+Paris
+possible
+San
+words
+York
+Exxtra
+btrie
diff --git a/packages/cspell-lib/fixtures/dictionaries/btrie/cspell.config.yaml b/packages/cspell-lib/fixtures/dictionaries/btrie/cspell.config.yaml
@@ -0,0 +1,22 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json
+
+dictionaryDefinitions:
+  - name: words
+    path: ./btrie-words.btrie.gz
+  - name: words-btrie
+    path: ./words.txt
+    btrie: ./btrie-words.btrie.gz
+
+dictionaries:
+  - words
+  - words-btrie
+
+overrides:
+  - filename: test.md
+    dictionaries:
+      - '!words'
+      - 'words-btrie'
+  - filename: test2.md
+    dictionaries:
+      - 'words'
+      - '!words-btrie'
diff --git a/packages/cspell-lib/fixtures/dictionaries/btrie/test.md b/packages/cspell-lib/fixtures/dictionaries/btrie/test.md
@@ -0,0 +1,3 @@
+# Test File
+
+This also has Exxtra words.
diff --git a/packages/cspell-lib/fixtures/dictionaries/btrie/test.txt b/packages/cspell-lib/fixtures/dictionaries/btrie/test.txt
@@ -0,0 +1,8 @@
+New York
+New Amsterdam
+Las Angles
+San Francisco
+New Delhi
+Mexico City
+London
+Paris
diff --git a/packages/cspell-lib/fixtures/dictionaries/btrie/test2.md b/packages/cspell-lib/fixtures/dictionaries/btrie/test2.md
@@ -0,0 +1,3 @@
+# Test File
+
+This also has Exxtra words.
diff --git a/packages/cspell-lib/fixtures/dictionaries/btrie/words.txt b/packages/cspell-lib/fixtures/dictionaries/btrie/words.txt
@@ -0,0 +1,28 @@
+Amsterdam
+Angles
+btrie
+cities
+City
+Configuration
+Default
+define
+Definitions
+Delhi
+dictionaries
+dictionary
+false
+file
+Francisco
+fully
+Inline
+Las
+load
+London
+Mexico
+name
+New
+Paris
+possible
+San
+words
+York
diff --git a/packages/cspell-lib/src/lib/Settings/DictionarySettings.ts b/packages/cspell-lib/src/lib/Settings/DictionarySettings.ts
@@ -54,10 +54,14 @@ function fixPath(def: DictionaryDefinitionInternal): DictionaryDefinitionInterna
         return def;
     }
     const newPath = fixDicPath(def.path, def.file);
+    const newBTriePath = def.btrie ? fixDicPath(def.btrie, def.file) : undefined;
+    // console.error('fixPath %o => %o', def.path, newPath);
+    // console.error('fixBTriePath %o => %o', def.btrie, newBTriePath);
     return {
         ...def,
         file: undefined,
         path: newPath,
+        btrie: newBTriePath,
     };
 }
 
@@ -178,6 +182,7 @@ class _DictionaryDefinitionInternalWithSource implements DictionaryFileDefinitio
         const {
             path: relPath = '',
             file = '',
+            btrie,
             addWords,
             description,
             dictionaryInformation,
@@ -194,12 +199,14 @@ class _DictionaryDefinitionInternalWithSource implements DictionaryFileDefinitio
         const name = determineName(filePath, def);
 
         const resolvedPath = toFilePathOrHref(resolveRelativeTo(filePath, defaultPath));
+        let bTriePath = btrie ? fixDicPath(btrie, file) : undefined;
+        bTriePath = bTriePath ? toFilePathOrHref(resolveRelativeTo(bTriePath, defaultPath)) : undefined;
 
         const ddi: DDI = {
             name,
             file: undefined,
             path: resolvedPath,
-            btrie: undefined,
+            btrie: bTriePath,
             addWords,
             description,
             dictionaryInformation,
diff --git a/packages/cspell-lib/src/lib/SpellingDictionary/DictionaryController/DictionaryLoader.ts b/packages/cspell-lib/src/lib/SpellingDictionary/DictionaryController/DictionaryLoader.ts
@@ -101,7 +101,7 @@ export class DictionaryLoader {
             if (entry) {
                 return entry.pending.then(([dictionary]) => dictionary);
             }
-            const loadedEntry = this.loadEntry(def.path, def);
+            const loadedEntry = this.loadEntry(def.btrie || def.path, def);
             this.setCacheEntry(key, loadedEntry, def);
             this.keepAliveCache.set(def, loadedEntry);
             return loadedEntry.pending.then(([dictionary]) => dictionary);
diff --git a/packages/cspell-lib/src/lib/test/english.spec.ts b/packages/cspell-lib/src/lib/test/english.spec.ts
@@ -21,14 +21,15 @@ describe('Validate English', async () => {
         const finalSettings = cspell.finalizeSettings(fileSettings);
         const dict = await getDictionary.getDictionary(finalSettings);
 
-        const startTime = process.hrtime();
+        const startTime = performance.now();
         // cspell:ignore installsallnecessary
         const results = dict.suggest(
             'installsallnecessary',
             createSuggestOptions(5, cspell.CompoundWordsMethod.SEPARATE_WORDS, 3),
         );
-        const elapsed = elapsedTimeMsFrom(startTime);
-        console.log(`Elapsed time ${elapsed.toFixed(2)}ms`);
+        const elapsed = performance.now() - startTime;
+        // console.log(`Elapsed time ${elapsed.toFixed(2)}ms`);
+        expect(elapsed).toBeLessThan(10_000);
         const sugs = results.map((a) => a.word);
         expect(sugs).toEqual(expect.arrayContaining(['installs all necessary']));
     });
@@ -101,11 +102,3 @@ describe('Validate English', async () => {
         expect(r.map((a) => a.text)).toEqual(['setsid', 'setsid']);
     });
 });
-
-function elapsedTimeMsFrom(relativeTo: [number, number]): number {
-    return hrTimeToMs(process.hrtime(relativeTo));
-}
-
-function hrTimeToMs(hrTime: [number, number]): number {
-    return hrTime[0] * 1.0e3 + hrTime[1] * 1.0e-6;
-}
diff --git a/packages/cspell-lib/src/lib/textValidation/docValidator.test.ts b/packages/cspell-lib/src/lib/textValidation/docValidator.test.ts
@@ -188,6 +188,7 @@ describe('docValidator', () => {
         filename                                               | maxDuplicateProblems | expectedIssues                    | expectedRawIssues
         ${tFix('issues/issue-5222/README.md')}                 | ${undefined}         | ${[]}                             | ${undefined}
         ${fix('sample-files-with-issues/WOX_Permissions.ps1')} | ${undefined}         | ${expectedIssues.WOX_Permissions} | ${expectedIssues.WOX_Permissions}
+        ${fixDict('btrie/README.md')}                          | ${undefined}         | ${[]}                             | ${undefined}
     `(
         'checkDocument group2 $filename $maxDuplicateProblems',
         async ({ filename, maxDuplicateProblems, expectedIssues, expectedRawIssues }) => {
diff --git a/packages/cspell/src/__snapshots__/app.test.ts.snap b/packages/cspell/src/__snapshots__/app.test.ts.snap
@@ -420,29 +420,33 @@ error	  Files Cached   :      0
 error
 error	  Processing Time: 0.00ms
 error
-error	Stats:
 error
-error	  __determineTextDocumentSettings: 0.00ms
 error
-error	  __finalizeSettings             : 0.00ms
+error	Perf Stats:
 error
-error	  __getDictionaryInternal        : 0.00ms
+error	  Name                            | Time (ms)
 error
-error	  __GlobMatcher                  : 0.00ms
+error	  prepareTimeMs                   | 0.00
 error
-error	  __shouldCheck                  : 0.00ms
+error	  __determineTextDocumentSettings | 0.00
 error
-error	  _checkDocument                 : 0.00ms
+error	  __getDictionaryInternal         | 0.00
 error
-error	  _prepTime                      : 0.00ms
+error	  __GlobMatcher                   | 0.00
 error
-error	  checkTimeMs                    : 0.00ms
+error	  __shouldCheck                   | 0.00
 error
-error	  loadTimeMs                     : 0.00ms
+error	  __finalizeSettings              | 0.00
 error
-error	  prepareTimeMs                  : 0.00ms
+error	  _prepTime                       | 0.00
 error
-error	  totalTimeMs                    : 0.00ms
+error	  checkTimeMs                     | 0.00
+error
+error	  _checkDocument                  | 0.00
+error
+error	  totalTimeMs                     | 0.00
+error
+error	  loadTimeMs                      | 0.00
 error"
 `;
 
diff --git a/packages/cspell/src/app.test.ts b/packages/cspell/src/app.test.ts
@@ -558,6 +558,7 @@ function makeLogger() {
         t = t.replaceAll(/ +[\d.]+ms\b/g, ' 0.00ms');
         t = t.replaceAll(/\b[\d.]+ms\b/g, '0.00ms');
         t = t.replaceAll(/\b[\d.]+S\b/g, '0.00S');
+        t = t.replaceAll(/\|\s+[\d]+\.\d\d\b/g, '| 0.00');
 
         const m = t.match(/.\[2K/g);
         if (m) {
diff --git a/packages/cspell/src/cli-reporter.ts b/packages/cspell/src/cli-reporter.ts
diff --git a/packages/cspell/src/util/perfMeasurements.ts b/packages/cspell/src/util/perfMeasurements.ts
diff --git a/packages/cspell/src/util/table.test.ts b/packages/cspell/src/util/table.test.ts
diff --git a/packages/cspell/src/util/table.ts b/packages/cspell/src/util/table.ts

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+# Test File`
	`2`	`+`
	`3`	`+This also has Exxtra words.`
Original file line number	Diff line number	Diff line change
`@@ -101,7 +101,7 @@ export class DictionaryLoader {`
`101`	`101`	`if (entry) {`
`102`	`102`	`return entry.pending.then(([dictionary]) => dictionary);`
`103`	`103`	`}`
`104`		`- const loadedEntry = this.loadEntry(def.path, def);`
	`104`	`+ const loadedEntry = this.loadEntry(def.btrie \|\| def.path, def);`
`105`	`105`	`this.setCacheEntry(key, loadedEntry, def);`
`106`	`106`	`this.keepAliveCache.set(def, loadedEntry);`
`107`	`107`	`return loadedEntry.pending.then(([dictionary]) => dictionary);`