fix: Make endian required when encoding a StringTable (#8265)

Signed-off-by: Jason Dent <[email protected]>
Co-authored-by: Copilot <[email protected]>

Author: Jason3S

cspell.json

@@ -78,7 +78,7 @@
         "test-packages/yarn/yarn2/test-eslint-plugin/**"
     ],
     "useGitignore": true,
-    "flagWords": [],
+    "flagWords": ["unit8:uint8", "Unit8:Uint8", "unit16:uint16", "Unit16:Uint16", "unit32:uint32", "Unit32:Uint32"],
     "ignoreWords": ["commitcomment"],
     "features": {
         "weighted-suggestions": true

packages/cspell-trie-lib/src/lib/StringTable/StringTable.ts

@@ -194,19 +194,36 @@ export class StringTableBuilder {
     }
 }
 
+/**
+ * The endian code used to identify endianness in the binary format.
+ * We use the 16-bit value 0x5453 (corresponding to the characters 'S' (0x53) and 'T' (0x54)).
+ * In little-endian representation, 0x5453 is stored as bytes 0x53 0x54 ('S', 'T').
+ * In big-endian representation, 0x5453 is stored as bytes 0x54 0x53 ('T', 'S').
+ *
+ * The value stored should match the value retrieved, otherwise the endianness is incorrect.
+ */
+const bomCode = 0x5453; // 16-bit BOM value used for endianness check
+
 function getStringTableBinaryFormat(): BinaryFormat {
     return new BinaryFormatBuilder()
         .addUint8('indexBits', 'The number of bits needed for each index entry', 32)
         .addUint8('strLenBits', 'The number of bits needed to store the max length of a string in the table.', 8)
-        .addString('reserved', 'Reserved for future use', 6)
+        .addUint16('bom', 'The Byte Order Mark.', bomCode)
+        .addString('reserved', 'Reserved for future use', 4)
         .addUint32ArrayPtr('index32', 'String index array of 32 bit entries')
         .addUint16ArrayPtr('index16', 'String index array of 16 bit entries', 'index32')
         .addUint8ArrayPtr('index', 'String index array of 8 bit entries', 'index32')
         .addUint8ArrayPtr('data', 'String byte data')
         .build();
 }
 
-export function encodeStringTableToBinary(table: StringTable, endian?: 'LE' | 'BE'): U8Array {
+/**
+ * Encodes a StringTable into binary data so that it can be stored or transmitted.
+ * @param table - the string table to encode
+ * @param endian - the resulting endianness of the data.
+ * @returns The encoded string table binary data.
+ */
+export function encodeStringTableToBinary(table: StringTable, endian: 'LE' | 'BE'): U8Array {
     const strLenBits = table.strLenBits;
     const offsetBits = Math.ceil(Math.log2(table.charData.length + 1));
     const minIndexBits = strLenBits + offsetBits;
@@ -218,6 +235,7 @@ export function encodeStringTableToBinary(table: StringTable, endian?: 'LE' | 'B
     const builder = new BinaryDataBuilder(format, endian);
     builder.setUint8('indexBits', indexBits);
     builder.setUint8('strLenBits', strLenBits);
+    builder.setUint16('bom', bomCode); // store the little endian value
     if (indexBits === 16) {
         builder.setPtrUint16Array('index16', toU16Array(table.index));
     } else {
@@ -228,13 +246,21 @@ export function encodeStringTableToBinary(table: StringTable, endian?: 'LE' | 'B
     return builder.build();
 }
 
-export function decodeStringTableFromBinary(data: U8Array, endian?: 'LE' | 'BE'): StringTable {
+/**
+ * Decodes binary data into a StringTable.
+ * @param data - the byte data of the string table.
+ * @param endian - the endianness of the encoded data.
+ * @returns The decoded StringTable.
+ */
+export function decodeStringTableFromBinary(data: U8Array, endian: 'LE' | 'BE'): StringTable {
     if (!data?.length) {
         return new StringTable([], new Uint8Array(0), 8);
     }
     const reader = new BinaryDataReader(data, getStringTableBinaryFormat(), endian);
     const indexBits = reader.getUint8('indexBits');
     const strLenBits = reader.getUint8('strLenBits');
+    const bomStored = reader.getUint16('bom');
+    assert(!bomStored || bomStored === bomCode, 'Endian mismatch');
     const index = indexBits === 16 ? reader.getPtrUint16Array('index16') : reader.getPtrUint32Array('index32');
     const buffer = reader.getPtrUint8Array('data');
     return new StringTable(index, buffer, strLenBits);

packages/cspell-trie-lib/src/lib/StringTable/__snapshots__/StringTable.test.ts.snap

@@ -1,7 +1,7 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
 exports[`StringTableBuilder > encode and decode StringTable 1`] = `
-"00000000  10 04 00 00  00 00 00 00  18 00 00 00  3a 00 00 00   ............:...
+"00000000  10 04 53 54  00 00 00 00  18 00 00 00  3a 00 00 00   ..ST........:...
 00000010  52 00 00 00  41 00 00 00  a5 02 f5 02  a2 02 a4 02   R...A...........
 00000020  51 02 03 02  02 00 a2 01  93 03 c4 01  c7 01 94 03   Q...............
 00000030  45 03 43 03  04 00 0a 00  d4 03 e3 03  36 02 37 02   E.C.........6.7.

packages/cspell-trie-lib/src/lib/TrieBlob/TrieBlobEncoder.ts

@@ -39,7 +39,7 @@ export function encodeTrieBlobToBTrie(blob: TrieBlobInfo): U8Array {
     builder.setString('characteristics', cvtTrieCharacteristicsToFlags(blob.characteristics));
 
     if (blob.stringTable.length) {
-        const stringTableData = encodeStringTableToBinary(blob.stringTable);
+        const stringTableData = encodeStringTableToBinary(blob.stringTable, builder.endian);
         builder.setPtrUint8Array('stringTable', stringTableData);
     }
 

packages/cspell-trie-lib/src/lib/TrieBlob/__snapshots__/TrieBlob.test.ts.snap

@@ -39,7 +39,7 @@ exports[`TrieBlob encode/decode > encode optimize hexDump 1`] = `
 00000080  00 00 00 00  05 00 00 00  61 07 00 00  62 08 00 00   ........a...b...
 00000090  67 09 00 00  6f 0a 00 00  73 0b 00 00  00 01 00 00   g...o...s.......
 000000a0  00 09 00 00  00 13 00 00  00 19 00 00  00 21 00 00   .............!..
-000000b0  00 33 00 00  10 04 00 00  00 00 00 00  18 00 00 00   .3..............
+000000b0  00 33 00 00  10 04 53 54  00 00 00 00  18 00 00 00   .3....ST........
 000000c0  34 00 00 00  4c 00 00 00  1b 00 00 00  00 00 51 00   4...L.........Q.
 000000d0  52 01 43 01  34 01 21 00  a2 00 93 00  a4 00 95 00   R.C.4.!.........
 000000e0  92 01 83 01  74 01 12 01  03 01 f4 00  e5 00 81 00   ....t...........

packages/cspell-trie-lib/src/lib/binary/binaryFormat.test.ts

@@ -169,6 +169,46 @@ describe('BinaryDataBuilder', () => {
         const readArr = reader.getPtrUint16Array('arrayPtr');
         expect(readArr).toEqual(arr);
     });
+
+    test('reader.getField', () => {
+        const format = new BinaryFormatBuilder()
+            .addString('header', 'The file header', 'Test Header')
+            .addUint16('value16', 'A uint16 value', 0x1234)
+            .addUint32ArrayPtr('arrayPtr', 'Pointer to uint32 array')
+            .build();
+        const builder = new BinaryDataBuilder(format);
+        const arr = new Uint32Array(numberRange(10, 20));
+        builder.setPtrUint32Array('arrayPtr', arr);
+        const data = builder.build();
+
+        const reader = new BinaryDataReader(data, format);
+
+        // make sure the field matches
+        const field = reader.getField('arrayPtr');
+        expect(field).toEqual(format.getField('arrayPtr'));
+    });
+
+    test('reader.getAsUint16', () => {
+        const format = new BinaryFormatBuilder()
+            .addString('header', 'The file header', 'Test Header')
+            .addUint16('data16', 'A 16-bit value', 0xabcd)
+            .addUint8Array('data8', 'An array of uint8', [0x12, 0x34])
+            .build();
+        const builder = new BinaryDataBuilder(format);
+        const data = builder.build();
+
+        const reader = new BinaryDataReader(data, format);
+
+        expect(reader.getUint16('data16')).toBe(0xabcd);
+
+        const expectedUint8Array = reader.endian === 'LE' ? new Uint8Array([0xcd, 0xab]) : new Uint8Array([0xab, 0xcd]);
+        const val8 = reader.getUint8Array('data16');
+        expect(val8).toEqual(expectedUint8Array);
+
+        const expectedUint16 = reader.endian === 'LE' ? 0x3412 : 0x1234;
+        const val16 = reader.getAsUint16('data8');
+        expect(val16).toBe(expectedUint16);
+    });
 });
 
 describe('field overrides', () => {

packages/cspell-trie-lib/src/lib/binary/binaryFormat.ts

@@ -172,6 +172,13 @@ export class BinaryFormatBuilder {
         return this;
     }
 
+    addUint8Array(name: string, description: string, length: number | Uint8Array | number[]): BinaryFormatBuilder {
+        // `as number` is needed because the type definition for `new Uint8Array` is wrong.
+        const value = new Uint8Array(length as number);
+        this.addData(name, description, 'value', value);
+        return this;
+    }
+
     addData(name: string, description: string, formatType: FormatType, data: DataArrayView): BinaryFormatBuilder {
         const byteSize = data.byteLength / data.length;
         assert(isByteAlignment(byteSize), `Invalid byte size: ${byteSize} for field: ${name}`);
@@ -636,6 +643,18 @@ export class BinaryDataReader {
         return view.getUint16(0, this.#useLE);
     }
 
+    /**
+     * Read a field as Uint16 starting at the given byte offset.
+     * @param name - name of field
+     * @param byteOffset - offset of in bytes from the beginning of the field
+     * @returns the value read.
+     */
+    getAsUint16(name: string, byteOffset: number = 0): number {
+        const element = this.getDataElement(name);
+        const view = new DataView(element.data.buffer, element.data.byteOffset, element.data.byteLength);
+        return view.getUint16(byteOffset, this.#useLE);
+    }
+
     /**
      * Get a Uint8 from the data.
      * @param name - name of the Uint8 field
@@ -754,9 +773,33 @@ export class BinaryDataReader {
         this.#useLE = endian === 'LE';
     }
 
+    get endian(): 'LE' | 'BE' {
+        return this.#useLE ? 'LE' : 'BE';
+    }
+
     reverseEndian(): void {
         this.#useLE = !this.#useLE;
     }
+
+    /**
+     * Get the raw bytes for a field.
+     * @param name - name of the field
+     * @returns the bytes or undefined
+     */
+    getUint8Array(name: string): U8Array | undefined {
+        const element = this.getDataElement(name);
+        if (!element) return undefined;
+        return element.data;
+    }
+
+    /**
+     * Get the FormatElement for a field.
+     * @param name - name of the field
+     * @returns the element or undefined
+     */
+    getField(name: string): FormatElement | undefined {
+        return this.format.getField(name);
+    }
 }
 
 function formatElementToJSON(fe: FormatElement): unknown {