feat: Added native CRC32 option. (#126)

Signed-off-by: Paolo Insogna <[email protected]>

Author: ShogunPanda

package.json

@@ -50,6 +50,7 @@
     "scule": "^1.3.0"
   },
   "optionalDependencies": {
+    "@node-rs/crc32": "^1.10.6",
     "lz4-napi": "^2.9.0",
     "snappy": "^7.3.3"
   },
@@ -81,4 +82,4 @@
   "engines": {
     "node": ">= 20.19.4 || >= 22.18.0 || >= 24.6.0"
   }
-}
+}

pnpm-lock.yaml

@@ -1,5 +1,5 @@
 // Based on the work from: https://github.com/tulios/kafkajs/blob/master/src/protocol/recordBatch/crc32C/crc32C.js
-
+import { createRequire } from 'node:module'
 import { DynamicBuffer } from './dynamic-buffer.ts'
 
 /* prettier-ignore */
@@ -70,7 +70,24 @@ const CRC: number[] = [
   0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351
 ]
 
-export function crc32c (data: Buffer | Uint8Array | DynamicBuffer): number {
+export function loadNativeCRC32C (): typeof jsCRC32C | null {
+  try {
+    const require = createRequire(import.meta.url)
+    const { crc32c: nativeImplementation } = require('@node-rs/crc32')
+
+    return function nativeCRC32C (data: Buffer | Uint8Array | DynamicBuffer): number {
+      const bytes: Uint8Array = DynamicBuffer.isDynamicBuffer(data)
+        ? ((data as DynamicBuffer).buffer as Uint8Array)
+        : new Uint8Array(data as Uint8Array)
+
+      return nativeImplementation(bytes)
+    }
+  } catch (error) {
+    return null
+  }
+}
+
+export function jsCRC32C (data: Buffer | Uint8Array | DynamicBuffer): number {
   const bytes: Uint8Array = DynamicBuffer.isDynamicBuffer(data)
     ? ((data as DynamicBuffer).buffer as Uint8Array)
     : new Uint8Array(data as Uint8Array)
@@ -83,3 +100,5 @@ export function crc32c (data: Buffer | Uint8Array | DynamicBuffer): number {
 
   return (crc ^ 0xffffffff) >>> 0
 }
+
+export const crc32c = loadNativeCRC32C() ?? jsCRC32C

src/protocol/crc32c.ts

@@ -1,6 +1,6 @@
 import { deepStrictEqual } from 'node:assert'
 import test from 'node:test'
-import { crc32c, DynamicBuffer } from '../../src/index.ts'
+import { DynamicBuffer, jsCRC32C, loadNativeCRC32C } from '../../src/index.ts'
 
 // Samples copied from https://github.com/tulios/kafkajs/blob/55b0b416308b9e597a5a6b97b0a6fd6b846255dc/src/protocol/recordBatch/crc32c/fixtures/samples.js
 const samples = [
@@ -68,29 +68,36 @@ const java = Buffer.from([
   57, 100, 102, 102, 99, 52, 97, 97, 99, 57, 2, 2, 97, 2, 98
 ])
 
-test('perform crc32c computations', () => {
-  const longString =
-    'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi mollis cursus metus vel tristique. Proin congue massa massa, a malesuada dolor ullamcorper a. Nulla eget leo vel orci venenatis placerat. Donec semper condimentum justo, vel sollicitudin dolor consequat id. Nunc sed aliquet felis, eget congue nisi. Mauris eu justo suscipit, elementum turpis ut, molestie tellus. Mauris ornare rutrum fringilla. Nulla dignissim luctus pretium. Nullam nec eros hendrerit sapien pellentesque sollicitudin. Integer eget ligula dui. Mauris nec cursus nibh. Nunc interdum elementum leo, eu sagittis eros sodales nec. Duis dictum nulla sed tincidunt malesuada. Quisque in vulputate sapien. Sed sit amet tellus a est porta rhoncus sed eu metus. Mauris non pulvinar nisl, volutpat luctus enim. Suspendisse est nisi, sagittis at risus quis, ultricies rhoncus sem. Donec ullamcorper purus eget sapien facilisis, eu eleifend felis viverra. Suspendisse elit neque, semper aliquet neque sed, egestas tempus leo. Duis condimentum turpis duis.'
-  const buffer = Buffer.from(longString)
-  deepStrictEqual(crc32c(new DynamicBuffer(buffer)), 1796588439)
-})
+const implementations = {
+  Javascript: jsCRC32C,
+  Native: loadNativeCRC32C()!
+}
 
-test('match the java crc32c code', () => {
-  deepStrictEqual(crc32c(new DynamicBuffer(java)), 818496390)
-  deepStrictEqual(crc32c(java), 818496390)
-})
+for (const [name, implementation] of Object.entries(implementations)) {
+  test(`perform crc32c computations (${name})`, () => {
+    const longString =
+      'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi mollis cursus metus vel tristique. Proin congue massa massa, a malesuada dolor ullamcorper a. Nulla eget leo vel orci venenatis placerat. Donec semper condimentum justo, vel sollicitudin dolor consequat id. Nunc sed aliquet felis, eget congue nisi. Mauris eu justo suscipit, elementum turpis ut, molestie tellus. Mauris ornare rutrum fringilla. Nulla dignissim luctus pretium. Nullam nec eros hendrerit sapien pellentesque sollicitudin. Integer eget ligula dui. Mauris nec cursus nibh. Nunc interdum elementum leo, eu sagittis eros sodales nec. Duis dictum nulla sed tincidunt malesuada. Quisque in vulputate sapien. Sed sit amet tellus a est porta rhoncus sed eu metus. Mauris non pulvinar nisl, volutpat luctus enim. Suspendisse est nisi, sagittis at risus quis, ultricies rhoncus sem. Donec ullamcorper purus eget sapien facilisis, eu eleifend felis viverra. Suspendisse elit neque, semper aliquet neque sed, egestas tempus leo. Duis condimentum turpis duis.'
+    const buffer = Buffer.from(longString)
+    deepStrictEqual(implementation(new DynamicBuffer(buffer)), 1796588439)
+  })
 
-test('samples', () => {
-  for (const sample of samples) {
-    const buffer = Buffer.from(sample.input)
-    deepStrictEqual(crc32c(new DynamicBuffer(buffer)), sample.output)
-  }
-})
+  test(`match the java crc32c code (${name})`, () => {
+    deepStrictEqual(implementation(new DynamicBuffer(java)), 818496390)
+    deepStrictEqual(implementation(java), 818496390)
+  })
+
+  test(`samples (${name})`, () => {
+    for (const sample of samples) {
+      const buffer = Buffer.from(sample.input)
+      deepStrictEqual(implementation(new DynamicBuffer(buffer)), sample.output)
+    }
+  })
 
-test('empty', () => {
-  deepStrictEqual(crc32c(new DynamicBuffer(Buffer.alloc(0))), 0)
-})
+  test(`empty (${name})`, () => {
+    deepStrictEqual(implementation(new DynamicBuffer(Buffer.alloc(0))), 0)
+  })
 
-test('unicode null', () => {
-  deepStrictEqual(crc32c(new DynamicBuffer(Buffer.from('\u0000'))), 1383945041)
-})
+  test(`unicode null (${name})`, () => {
+    deepStrictEqual(implementation(new DynamicBuffer(Buffer.from('\u0000'))), 1383945041)
+  })
+}