Small code simplifications and add tests (#22)

Author: cmdcolin

jest.config.js

@@ -2,4 +2,4 @@
 module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
-};
+}

package.json

@@ -20,24 +20,25 @@
     "preversion": "yarn lint && yarn test --run && yarn build",
     "postbuild:es5": "echo '{\"type\": \"commonjs\"}' > dist/package.json",
     "postversion": "git push --follow-tags",
-    "test": "vitest"
+    "test": "vitest",
+    "format": "prettier --write ."
   },
   "name": "@gmod/trix",
   "author": "Matt Morgan",
   "repository": "GMOD/trix-js",
   "devDependencies": {
-    "@types/node": "^22.15.21",
-    "@typescript-eslint/eslint-plugin": "^8.4.0",
-    "@typescript-eslint/parser": "^8.4.0",
+    "@types/node": "^25.0.9",
+    "@typescript-eslint/eslint-plugin": "^8.53.0",
+    "@typescript-eslint/parser": "^8.53.0",
     "eslint": "^9.0.0",
     "eslint-plugin-import": "^2.31.0",
     "eslint-plugin-unicorn": "^62.0.0",
-    "generic-filehandle2": "^2.0.5",
-    "prettier": "^3.3.3",
+    "generic-filehandle2": "^2.0.18",
+    "prettier": "^3.8.0",
     "rimraf": "^6.0.1",
     "typescript": "^5.7.0",
-    "typescript-eslint": "^8.4.0",
-    "vitest": "^4.0.6"
+    "typescript-eslint": "^8.53.0",
+    "vitest": "^4.0.17"
   },
   "publishConfig": {
     "access": "public"

src/index.ts

@@ -10,6 +10,9 @@ const CHUNK_SIZE = 65536
 const ADDRESS_SIZE = 10
 
 export default class Trix {
+  private decoder = new TextDecoder('utf8')
+  private indexCache?: readonly (readonly [string, number])[]
+
   constructor(
     public ixxFile: GenericFilehandle,
     public ixFile: GenericFilehandle,
@@ -19,70 +22,61 @@ export default class Trix {
   async search(searchString: string, opts?: { signal?: AbortSignal }) {
     let resultArr = [] as [string, string][]
     const searchWords = searchString.split(' ')
+    const firstWord = searchWords[0]
+
+    // validate that we have a non-empty search term
+    if (firstWord) {
+      const searchWord = firstWord.toLowerCase()
+      const res = await this.getBuffer(searchWord, opts)
+
+      if (res) {
+        let { end, buffer } = res
+        let done = false
+        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+        while (!done) {
+          const str = this.decoder.decode(buffer)
+
+          // slice to lastIndexOf('\n') to make sure we get complete records
+          // since the buffer fetch could get halfway into a record
+          const lines = str
+            .slice(0, str.lastIndexOf('\n'))
+            .split('\n')
+            .filter(f => f)
+
+          const hits2 = [] as string[]
+          for (const line of lines) {
+            const word = line.split(' ')[0]
+
+            if (word.startsWith(searchWord)) {
+              hits2.push(line)
+            } else if (word > searchWord) {
+              // we are done scanning if we are lexicographically greater than
+              // the search string
+              done = true
+            }
+          }
+          const hits = hits2.flatMap(line => {
+            const [term, ...parts] = line.split(' ')
+            return parts
+              .filter(elt => elt)
+              .map(elt => [term, elt.split(',')[0]] as [string, string])
+          })
 
-    // we only search one word at a time
-    const searchWord = searchWords[0].toLowerCase()
-    const res = await this._getBuffer(searchWord, opts)
-    if (!res) {
-      return []
-    }
-
-    let { end, buffer } = res
-    let done = false
-    const decoder = new TextDecoder('utf8')
-    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-    while (!done) {
-      let foundSomething = false
-      const str = decoder.decode(buffer)
-
-      // slice to lastIndexOf('\n') to make sure we get complete records
-      // since the buffer fetch could get halfway into a record
-      const lines = str
-        .slice(0, str.lastIndexOf('\n'))
-        .split('\n')
-        .filter(f => !!f)
-
-      const hits2 = [] as string[]
-      for (const line of lines) {
-        const word = line.split(' ')[0]
-        const match = word.startsWith(searchWord)
-        if (!foundSomething && match) {
-          foundSomething = true
-        }
-
-        // we are done scanning if we are lexicographically greater than the
-        // search string
-        if (word.slice(0, searchWord.length) > searchWord) {
-          done = true
-        }
-        if (match) {
-          hits2.push(line)
-        }
-      }
-      const hits = hits2.flatMap(line => {
-        const [term, ...parts] = line.split(' ')
-        return parts.map(elt => [term, elt.split(',')[0]] as [string, string])
-      })
-
-      // if we are not done, and we haven't filled up maxResults with hits yet,
-      // then refetch
-      if (resultArr.length + hits.length < this.maxResults && !done) {
-        const res2 = await this.ixFile.read(CHUNK_SIZE, end, opts)
-
-        // early break if empty response
-        if (res2.length === 0) {
           resultArr = resultArr.concat(hits)
-          break
-        }
-        buffer = concatUint8Array([buffer, res2])
-        end += CHUNK_SIZE
-      }
 
-      // if we have filled up the hits, or we are detected to be done via the
-      // filtering, then return
-      else if (resultArr.length + hits.length >= this.maxResults || done) {
-        resultArr = resultArr.concat(hits)
-        break
+          // if we are done or have filled up maxResults, break
+          if (done || resultArr.length >= this.maxResults) {
+            break
+          }
+
+          // fetch more data
+          const res2 = await this.ixFile.read(CHUNK_SIZE, end, opts)
+          if (res2.length === 0) {
+            break
+          }
+          buffer = concatUint8Array([buffer, res2])
+          end += CHUNK_SIZE
+        }
       }
     }
 
@@ -91,46 +85,40 @@ export default class Trix {
   }
 
   private async getIndex(opts?: { signal?: AbortSignal }) {
+    if (this.indexCache) {
+      return this.indexCache
+    }
     const file = await this.ixxFile.readFile({
       encoding: 'utf8',
       ...opts,
     })
-    return file
+    const result = file
       .split('\n')
-      .filter(f => !!f)
+      .filter(f => f)
       .map(line => {
         const p = line.length - ADDRESS_SIZE
         const prefix = line.slice(0, p)
         const posStr = line.slice(p)
         const pos = Number.parseInt(posStr, 16)
         return [prefix, pos] as const
       })
+    this.indexCache = result
+    return result
   }
 
-  private async _getBuffer(
-    searchWord: string,
-    opts?: { signal?: AbortSignal },
-  ) {
+  private async getBuffer(searchWord: string, opts?: { signal?: AbortSignal }) {
     let start = 0
-    let end = 65536
+    let end = CHUNK_SIZE
     const indexes = await this.getIndex(opts)
     for (const [key, value] of indexes) {
       const trimmedKey = key.slice(0, searchWord.length)
       if (trimmedKey < searchWord) {
         start = value
-        end = value + 65536
+        end = value + CHUNK_SIZE
       }
     }
 
-    // Return the buffer and its end position in the file.
-    const len = end - start
-    if (len < 0) {
-      return undefined
-    }
-    const buffer = await this.ixFile.read(len, start, opts)
-    return {
-      buffer,
-      end,
-    }
+    const buffer = await this.ixFile.read(end - start, start, opts)
+    return { buffer, end }
   }
 }

src/util.ts

@@ -1,9 +1,9 @@
 export function sum(array: Uint8Array[]) {
-  let sum = 0
+  let total = 0
   for (const entry of array) {
-    sum += entry.length
+    total += entry.length
   }
-  return sum
+  return total
 }
 export function concatUint8Array(args: Uint8Array[]) {
   const mergedArray = new Uint8Array(sum(args))

test/__snapshots__/edge-cases.test.ts.snap

@@ -0,0 +1,18 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Edge case handling > handles search term with trailing spaces 1`] = `
+[
+  [
+    "this",
+    "id1",
+  ],
+  [
+    "this",
+    "id2",
+  ],
+  [
+    "this",
+    "id3",
+  ],
+]
+`;

test/dedupe.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, it } from 'vitest'
+
+import { dedupe } from '../src/dedupe.ts'
+
+describe('dedupe', () => {
+  it('returns empty array for empty input', () => {
+    expect(dedupe([])).toEqual([])
+  })
+
+  it('removes duplicate primitives', () => {
+    expect(dedupe([1, 2, 2, 3, 3, 3])).toEqual([1, 2, 3])
+  })
+
+  it('removes duplicate strings', () => {
+    expect(dedupe(['a', 'b', 'a', 'c', 'b'])).toEqual(['a', 'b', 'c'])
+  })
+
+  it('removes duplicate objects using default hasher', () => {
+    const input = [{ x: 1 }, { x: 2 }, { x: 1 }, { x: 3 }]
+    expect(dedupe(input)).toEqual([{ x: 1 }, { x: 2 }, { x: 3 }])
+  })
+
+  it('uses custom hasher function', () => {
+    const input = [
+      { id: 1, name: 'a' },
+      { id: 2, name: 'b' },
+      { id: 1, name: 'c' },
+    ]
+    const result = dedupe(input, item => String(item.id))
+    expect(result).toEqual([
+      { id: 1, name: 'a' },
+      { id: 2, name: 'b' },
+    ])
+  })
+
+  it('preserves order of first occurrence', () => {
+    expect(dedupe([3, 1, 2, 1, 3, 2])).toEqual([3, 1, 2])
+  })
+
+  it('handles arrays with tuple values', () => {
+    const input: [string, string][] = [
+      ['term1', 'value1'],
+      ['term2', 'value2'],
+      ['term1', 'value1'],
+      ['term3', 'value1'],
+    ]
+    const result = dedupe(input, elt => elt[1])
+    expect(result).toEqual([
+      ['term1', 'value1'],
+      ['term2', 'value2'],
+    ])
+  })
+})