feat(NODE-6540): Add c++ zstd compression API

.github/workflows/lint.yml

@@ -0,0 +1,38 @@
+name: Lint
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    name: ${{ matrix.lint-target }}
+    strategy:
+      matrix:
+        lint-target: ["c++", "typescript"]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Use Node.js LTS
+      uses: actions/setup-node@v4
+      with:
+        node-version: 'lts/*'
+        cache: 'npm'
+
+    - name: Install dependencies
+      shell: bash
+      run: npm i --ignore-scripts
+
+    - if: matrix.lint-target == 'c++'
+      shell: bash
+      run: |
+        npm run check:clang-format
+    - if: matrix.lint-target == 'typescript'
+      shell: bash
+      run: |
+        npm run check:eslint

.github/workflows/test.yml

@@ -24,8 +24,16 @@ jobs:
           cache: 'npm'
           registry-url: 'https://registry.npmjs.org'
 
-      - name: Build with Node.js ${{ matrix.node }} on ${{ matrix.os }}
-        run: npm install && npm run compile
+      - name: Install zstd
+        run: npm run install-zstd
+        shell: bash
+
+      - name: install dependencies
+        run: npm install --loglevel verbose --ignore-scripts
+        shell: bash
+
+      - name: Compile addon
+        run: npm run compile
         shell: bash
 
       - name: Test ${{ matrix.os }}

.gitignore

@@ -18,3 +18,4 @@ node_modules
 build
 
 npm-debug.log
+deps

addon/compress.h

@@ -0,0 +1,21 @@
+#include <vector>
+
+#include "compression_worker.h"  // CompressionResult
+#include "zstd.h"
+
+CompressionResult compress(const std::vector<uint8_t> data, size_t compression_level) {
+    size_t output_buffer_size = ZSTD_compressBound(data.size());
+    std::vector<uint8_t> output(output_buffer_size);
+
+    size_t result_code =
+        ZSTD_compress(output.data(), output.size(), data.data(), data.size(), compression_level);
+
+    if (ZSTD_isError(result_code)) {
+        std::string error(ZSTD_getErrorName(result_code));
+        return CompressionResult::Error(error);
+    }
+
+    output.resize(result_code);
+
+    return CompressionResult::Ok(output);
+}

addon/compression_worker.h

@@ -0,0 +1,99 @@
+#ifndef COMPRESSION_WORKER_H
+#define COMPRESSION_WORKER_H
+#include <napi.h>
+
+#include <optional>
+
+using namespace Napi;
+
+/**
+ * @brief A class that represents the result of a compression operation.  Once the
+ * MACOS_DEPLOYMENT_TARGET can be raised to 10.13 and use a c++17, we can remove this class and use
+ * a std::optional<std::variant<std::vector<uint8_t>, std::string>>> instead.
+ */
+struct CompressionResult {
+    CompressionResult(std::string error,
+                      std::vector<uint8_t> result,
+                      bool hasError,
+                      bool hasResult,
+                      bool initialized)
+        : error(error),
+          result(result),
+          hasError(hasError),
+          hasResult(hasResult),
+          initialized(true) {}
+
+   public:
+    static CompressionResult Error(std::string error) {
+        return CompressionResult(error, std::vector<uint8_t>(), true, false, true);
+    }
+
+    static CompressionResult Ok(std::vector<uint8_t> result) {
+        return CompressionResult(std::string(""), result, false, true, true);
+    }
+
+    static CompressionResult Empty() {
+        return CompressionResult(std::string(""), std::vector<uint8_t>(), false, false, false);
+    }
+
+    std::string error;
+    std::vector<uint8_t> result;
+
+    bool hasError;
+    bool hasResult;
+    bool initialized;
+};
+
+/**
+ * @brief An asynchronous Napi::Worker that can be with any function that produces
+ * CompressionResults.
+ * */
+class CompressionWorker : public Napi::AsyncWorker {
+   public:
+    CompressionWorker(const Napi::Env& env, std::function<CompressionResult()> worker)
+        : Napi::AsyncWorker{env, "Worker"},
+          m_deferred{env},
+          worker(worker),
+          result(CompressionResult::Empty()) {}
+
+    Napi::Promise GetPromise() {
+        return m_deferred.Promise();
+    }
+
+   protected:
+    void Execute() {
+        result = worker();
+    }
+
+    void OnOK() {
+        if (!result.initialized) {
+            m_deferred.Reject(Napi::Error::New(Env(),
+                                               "zstd runtime error - async worker finished without "
+                                               "a compression or decompression result.")
+                                  .Value());
+        } else if (result.hasError) {
+            m_deferred.Reject(Napi::Error::New(Env(), result.error).Value());
+        } else if (result.hasResult) {
+            Buffer<uint8_t> output =
+                Buffer<uint8_t>::Copy(m_deferred.Env(), result.result.data(), result.result.size());
+
+            m_deferred.Resolve(output);
+        } else {
+            m_deferred.Reject(Napi::Error::New(Env(),
+                                               "zstd runtime error - async worker finished without "
+                                               "a compression or decompression result.")
+                                  .Value());
+        }
+    }
+
+    void OnError(const Napi::Error& err) {
+        m_deferred.Reject(err.Value());
+    }
+
+   private:
+    Napi::Promise::Deferred m_deferred;
+    std::function<CompressionResult()> worker;
+    CompressionResult result;
+};
+
+#endif

addon/decompress.h

@@ -0,0 +1,31 @@
+#include <vector>
+
+#include "compression_worker.h"  // CompressionResult
+#include "zstd.h"
+
+CompressionResult decompress(const std::vector<uint8_t>& compressed) {
+    std::vector<uint8_t> decompressed;
+
+    using DCTX_Deleter = void (*)(ZSTD_DCtx*);
+
+    std::unique_ptr<ZSTD_DCtx, DCTX_Deleter> decompression_context(ZSTD_createDCtx(),
+                                                                   (DCTX_Deleter)ZSTD_freeDCtx);
+
+    ZSTD_inBuffer input = {compressed.data(), compressed.size(), 0};
+
+    while (input.pos < input.size) {
+        std::vector<uint8_t> chunk(ZSTD_DStreamOutSize());
+        ZSTD_outBuffer output = {chunk.data(), chunk.size(), 0};
+        size_t const ret = ZSTD_decompressStream(decompression_context.get(), &output, &input);
+        if (ZSTD_isError(ret)) {
+            std::string error(ZSTD_getErrorName(ret));
+            return CompressionResult::Error(error);
+        }
+
+        for (size_t i = 0; i < output.pos; ++i) {
+            decompressed.push_back(chunk[i]);
+        }
+    }
+
+    return CompressionResult::Ok(decompressed);
+}

addon/napi_utils.h

@@ -0,0 +1,40 @@
+#ifndef NAPI_UTILS_H
+#define NAPI_UTILS_H
+
+#include <napi.h>
+
+using namespace Napi;
+
+/**
+ * @brief Get the Bytes From Uint8 Array object
+ *
+ * this function copies the bytes out of the Uint8Array.
+ */
+std::vector<uint8_t> getBytesFromUint8Array(const Uint8Array& source) {
+    const uint8_t* input_data = source.Data();
+    size_t total = source.ElementLength();
+    std::vector<uint8_t> data(total);
+
+    std::copy(input_data, input_data + total, data.data());
+
+    return data;
+}
+
+/**
+ * @brief Given an Napi::Value, this function returns the value as a Uint8Array, if the
+ * Value is a Uint8Array. Otherwise, this function throws.
+ *
+ * @param v - An Napi::Value
+ * @param argument_name - the name of the value, to use when constructing an error message.
+ * @return Napi::Uint8Array
+ */
+Uint8Array Uint8ArrayFromValue(Value v, std::string argument_name) {
+    if (!v.IsTypedArray() || v.As<TypedArray>().TypedArrayType() != napi_uint8_array) {
+        std::string error_message = "Parameter `" + argument_name + "` must be a Uint8Array.";
+        throw TypeError::New(v.Env(), error_message);
+    }
+
+    return v.As<Uint8Array>();
+}
+
+#endif

addon/zstd.cpp

@@ -1,14 +1,53 @@
+#include "zstd.h"
+
 #include <napi.h>
 
+#include <string>
+#include <vector>
+
+#include "compress.h"
+#include "compression_worker.h"
+#include "decompress.h"
+#include "napi_utils.h"
+
 using namespace Napi;
 
-Napi::String Compress(const Napi::CallbackInfo& info) {
-    auto string = Napi::String::New(info.Env(), "compress()");
-    return string;
+Napi::Promise Compress(const Napi::CallbackInfo& info) {
+    // Argument handling happens in JS
+    if (info.Length() != 2) {
+        std::string error_message = "Expected two arguments.";
+        throw TypeError::New(info.Env(), error_message);
+    }
+
+    Uint8Array to_compress = Uint8ArrayFromValue(info[0], "buffer");
+    std::vector<uint8_t> data = getBytesFromUint8Array(to_compress);
+
+    size_t compression_level = (size_t)info[1].ToNumber().Int32Value();
+
+    CompressionWorker* worker = new CompressionWorker(
+        info.Env(),
+        [data = std::move(data), compression_level] { return compress(data, compression_level); });
+
+    worker->Queue();
+
+    return worker->GetPromise();
 }
-Napi::String Decompress(const Napi::CallbackInfo& info) {
-    auto string = Napi::String::New(info.Env(), "decompress()");
-    return string;
+
+Napi::Promise Decompress(const CallbackInfo& info) {
+    // Argument handling happens in JS
+    if (info.Length() != 1) {
+        std::string error_message = "Expected one argument.";
+        throw TypeError::New(info.Env(), error_message);
+    }
+
+    Napi::Uint8Array compressed_data = Uint8ArrayFromValue(info[0], "buffer");
+    std::vector<uint8_t> data = getBytesFromUint8Array(compressed_data);
+    CompressionWorker* worker =
+        new CompressionWorker(info.Env(), [data = std::move(data)] { return decompress(data); });
+
+    worker->Queue();
+
+    return worker->GetPromise();
 }
 
 Napi::Object Init(Napi::Env env, Napi::Object exports) {

binding.gyp

@@ -4,14 +4,19 @@
     'type': 'loadable_module',
     'defines': ['ZSTD_STATIC_LINKING_ONLY'],
     'include_dirs': [
-        "<!(node -p \"require('node-addon-api').include_dir\")"
+        "<!(node -p \"require('node-addon-api').include_dir\")",
+        "<(module_root_dir)/deps/zstd/lib",
     ],
     'variables': {
       'ARCH': '<(host_arch)',
       'built_with_electron%': 0
     },
     'sources': [
-      'addon/zstd.cpp'
+      'addon/zstd.cpp',
+      'addon/compression_worker.h',
+      'addon/compress.h',
+      'addon/decompress.h',
+      'addon/napi_utils.h',
     ],
     'xcode_settings': {
       'GCC_ENABLE_CPP_EXCEPTIONS': 'YES',
@@ -23,6 +28,11 @@
     'cflags_cc!': [ '-fno-exceptions' ],
     'msvs_settings': {
       'VCCLCompilerTool': { 'ExceptionHandling': 1 },
-    }
+    },
+	 'link_settings': {
+      'libraries': [
+        '<(module_root_dir)/deps/zstd/build/cmake/lib/libzstd.a',
+      ]
+    },
   }]
 }

etc/install-zstd.sh

@@ -0,0 +1,26 @@
+
+set -o xtrace
+
+clean_deps() {
+	rm -rf deps
+}
+
+download_zstd() {
+	rm -rf deps
+	mkdir -p deps/zstd
+
+	curl -L "https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-1.5.6.tar.gz" \
+	 	| tar  -zxf - -C deps/zstd --strip-components 1
+}
+
+build_zstd() {
+	export MACOSX_DEPLOYMENT_TARGET=10.12
+	cd deps/zstd/build/cmake
+
+	cmake .
+	make
+}
+
+clean_deps
+download_zstd
+build_zstd