From 2fed9fa7fda73f3050b58cd16d5e954cdb464678 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 4 Dec 2024 12:42:31 -0700 Subject: [PATCH 1/9] wip --- etc/install-zstd.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/install-zstd.sh b/etc/install-zstd.sh index d9ba539..6e1e9de 100644 --- a/etc/install-zstd.sh +++ b/etc/install-zstd.sh @@ -1,5 +1,6 @@ #!/bin/sh set -o xtrace +set -o errexit clean_deps() { rm -rf deps From 0f74d7b6123d158823398ef74408c53a14713018 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 4 Dec 2024 12:44:11 -0700 Subject: [PATCH 2/9] make matrix smaller to save CI --- .github/workflows/test.yml | 157 +++++++++++++++++++------------------ 1 file changed, 79 insertions(+), 78 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 57413b7..9b4901b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,8 @@ jobs: strategy: matrix: os: [macos-latest, windows-2019, macos-13] - node: [16.20.1, 18.x, 20.x, 22.x] + # node: [16.20.1, 18.x, 20.x, 22.x] + node: [16.20.1] fail-fast: false runs-on: ${{ matrix.os }} steps: @@ -36,80 +37,80 @@ jobs: shell: bash run: npm test - container_tests_glibc: - runs-on: ubuntu-latest - strategy: - matrix: - linux_arch: [s390x, arm64, amd64] - node: [16.x, 18.x, 20.x, 22.x] - fail-fast: false - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node }} - - - name: Get Full Node.js Version - id: get_nodejs_version - shell: bash - run: | - echo "version=$(node --print 'process.version.slice(1)')" >> "$GITHUB_OUTPUT" - echo "ubuntu_version=$(node --print '(+process.version.slice(1).split(`.`).at(0)) > 16 ? `noble` : `bionic`')" >> "$GITHUB_OUTPUT" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Run Buildx - run: | - docker buildx create --name builder --bootstrap --use - docker buildx build \ - --platform linux/${{ matrix.linux_arch }} \ - --build-arg="NODE_ARCH=${{ matrix.linux_arch == 'amd64' && 'x64' || matrix.linux_arch }}" \ - --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ - --build-arg="UBUNTU_VERSION=${{ steps.get_nodejs_version.outputs.ubuntu_version }}" \ - --build-arg="RUN_TEST=true" \ - --output type=local,dest=./prebuilds,platform-split=false \ - -f ./.github/docker/Dockerfile.glibc \ - . - - container_tests_musl: - runs-on: ubuntu-latest - strategy: - matrix: - linux_arch: [amd64, arm64] - node: [16.20.1, 18.x, 20.x, 22.x] - fail-fast: false - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node }} - - - name: Get Full Node.js Version - id: get_nodejs_version - shell: bash - run: | - echo "version=$(node --print 'process.version.slice(1)')" >> "$GITHUB_OUTPUT" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Run Buildx - run: | - docker buildx create --name builder --bootstrap --use - docker --debug buildx build --progress=plain --no-cache \ - --platform linux/${{ matrix.linux_arch }} \ - --build-arg="PLATFORM=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}" \ - --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ - --build-arg="RUN_TEST=true" \ - --output type=local,dest=./prebuilds,platform-split=false \ - -f ./.github/docker/Dockerfile.musl \ - . + # container_tests_glibc: + # runs-on: ubuntu-latest + # strategy: + # matrix: + # linux_arch: [s390x, arm64, amd64] + # node: [16.x, 18.x, 20.x, 22.x] + # fail-fast: false + # steps: + # - uses: actions/checkout@v4 + + # - uses: actions/setup-node@v4 + # with: + # node-version: ${{ matrix.node }} + + # - name: Get Full Node.js Version + # id: get_nodejs_version + # shell: bash + # run: | + # echo "version=$(node --print 'process.version.slice(1)')" >> "$GITHUB_OUTPUT" + # echo "ubuntu_version=$(node --print '(+process.version.slice(1).split(`.`).at(0)) > 16 ? `noble` : `bionic`')" >> "$GITHUB_OUTPUT" + + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v3 + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + + # - name: Run Buildx + # run: | + # docker buildx create --name builder --bootstrap --use + # docker buildx build \ + # --platform linux/${{ matrix.linux_arch }} \ + # --build-arg="NODE_ARCH=${{ matrix.linux_arch == 'amd64' && 'x64' || matrix.linux_arch }}" \ + # --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ + # --build-arg="UBUNTU_VERSION=${{ steps.get_nodejs_version.outputs.ubuntu_version }}" \ + # --build-arg="RUN_TEST=true" \ + # --output type=local,dest=./prebuilds,platform-split=false \ + # -f ./.github/docker/Dockerfile.glibc \ + # . + + # container_tests_musl: + # runs-on: ubuntu-latest + # strategy: + # matrix: + # linux_arch: [amd64, arm64] + # node: [16.20.1, 18.x, 20.x, 22.x] + # fail-fast: false + # steps: + # - uses: actions/checkout@v4 + + # - uses: actions/setup-node@v4 + # with: + # node-version: ${{ matrix.node }} + + # - name: Get Full Node.js Version + # id: get_nodejs_version + # shell: bash + # run: | + # echo "version=$(node --print 'process.version.slice(1)')" >> "$GITHUB_OUTPUT" + + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v3 + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + + # - name: Run Buildx + # run: | + # docker buildx create --name builder --bootstrap --use + # docker --debug buildx build --progress=plain --no-cache \ + # --platform linux/${{ matrix.linux_arch }} \ + # --build-arg="PLATFORM=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}" \ + # --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ + # --build-arg="RUN_TEST=true" \ + # --output type=local,dest=./prebuilds,platform-split=false \ + # -f ./.github/docker/Dockerfile.musl \ + # . From 52dd4e380bd2c5a1500f38fc3b2d921a796e35a0 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 4 Dec 2024 12:59:41 -0700 Subject: [PATCH 3/9] test test --- etc/install-zstd.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/etc/install-zstd.sh b/etc/install-zstd.sh index 6e1e9de..9c472ce 100644 --- a/etc/install-zstd.sh +++ b/etc/install-zstd.sh @@ -6,12 +6,23 @@ clean_deps() { rm -rf deps } +download_zstd_windows() { + curl -L "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION-win64.zip" \ + | tar -zxf - -C deps/zstd --strip-components 1 +} + download_zstd() { mkdir -p deps/zstd ZSTD_VERSION=$(node -p "require('./package.json')['mongodb:zstd_version']") + is_windows=$(node -p "process.platform === 'win32'") + - curl -L "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz" \ + if [ "$is_windows" == "true" ]; then + download_zstd_windows + else + curl -L "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz" \ | tar -zxf - -C deps/zstd --strip-components 1 + fi } build_zstd() { From 2e204342fdd43c6d8b49068909b8b04ea1870c89 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 4 Dec 2024 13:04:02 -0700 Subject: [PATCH 4/9] use unzip --- etc/install-zstd.sh | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/etc/install-zstd.sh b/etc/install-zstd.sh index 9c472ce..851aa1b 100644 --- a/etc/install-zstd.sh +++ b/etc/install-zstd.sh @@ -6,22 +6,17 @@ clean_deps() { rm -rf deps } -download_zstd_windows() { - curl -L "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION-win64.zip" \ - | tar -zxf - -C deps/zstd --strip-components 1 -} - download_zstd() { mkdir -p deps/zstd ZSTD_VERSION=$(node -p "require('./package.json')['mongodb:zstd_version']") is_windows=$(node -p "process.platform === 'win32'") - if [ "$is_windows" == "true" ]; then - download_zstd_windows + curl -L "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION-win64.zip" \ + | unzip - -C deps/zstd else curl -L "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz" \ - | tar -zxf - -C deps/zstd --strip-components 1 + | tar -zxf - -C deps/zstd --strip-components 1 fi } From e870ac4e8252be600a703c89177641aab9ec548b Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 4 Dec 2024 13:11:12 -0700 Subject: [PATCH 5/9] working unzip? --- etc/install-zstd.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/etc/install-zstd.sh b/etc/install-zstd.sh index 851aa1b..da9d26d 100644 --- a/etc/install-zstd.sh +++ b/etc/install-zstd.sh @@ -11,9 +11,12 @@ download_zstd() { ZSTD_VERSION=$(node -p "require('./package.json')['mongodb:zstd_version']") is_windows=$(node -p "process.platform === 'win32'") - if [ "$is_windows" == "true" ]; then - curl -L "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION-win64.zip" \ - | unzip - -C deps/zstd + if [ "$is_windows" == "false" ]; then + curl -L -o zstd.zip "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-v$ZSTD_VERSION-win64.zip" + unzip zstd.zip + cp -r zstd-v$ZSTD_VERSION-win64/* deps/zstd + rm zstd.zip + rm -rf zstd-v$ZSTD_VERSION-win64 else curl -L "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz" \ | tar -zxf - -C deps/zstd --strip-components 1 @@ -32,4 +35,4 @@ build_zstd() { clean_deps download_zstd -build_zstd \ No newline at end of file +# build_zstd \ No newline at end of file From 04972b7c80f1731c6a034d51ac69a19287cd65d7 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 4 Dec 2024 13:16:45 -0700 Subject: [PATCH 6/9] windows fix? --- binding.gyp | 5 ++++- etc/install-zstd.sh | 25 +++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/binding.gyp b/binding.gyp index 3da1b99..75c6fec 100644 --- a/binding.gyp +++ b/binding.gyp @@ -23,9 +23,12 @@ { 'link_settings': { 'libraries': [ - '<(module_root_dir)/deps/zstd/build/cmake/lib/Debug/zstd_static.lib' + '<(module_root_dir)/deps/zstd/static/zstd_static.lib' ] }, + 'include_dirs': [ + '<(module_root_dir)/deps/zstd/include' + ], }, { # macos and linux 'link_settings': { diff --git a/etc/install-zstd.sh b/etc/install-zstd.sh index da9d26d..7fb4eaa 100644 --- a/etc/install-zstd.sh +++ b/etc/install-zstd.sh @@ -6,20 +6,29 @@ clean_deps() { rm -rf deps } +download_windows() { + # windows does not support symlinks, so we must download the `win64` build specifically + curl -L -o zstd.zip "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-v$ZSTD_VERSION-win64.zip" + # unlike tar, unzip does not have a "strip components" option. so we unzip the file, copy all the contents to the correct location, + # and then delete both the .zip and the unziped content. + unzip zstd.zip + cp -r zstd-v$ZSTD_VERSION-win64/* deps/zstd + rm zstd.zip + rm -rf zstd-v$ZSTD_VERSION-win64 +} + download_zstd() { mkdir -p deps/zstd ZSTD_VERSION=$(node -p "require('./package.json')['mongodb:zstd_version']") is_windows=$(node -p "process.platform === 'win32'") - if [ "$is_windows" == "false" ]; then - curl -L -o zstd.zip "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-v$ZSTD_VERSION-win64.zip" - unzip zstd.zip - cp -r zstd-v$ZSTD_VERSION-win64/* deps/zstd - rm zstd.zip - rm -rf zstd-v$ZSTD_VERSION-win64 + if [ "$is_windows" == "true" ]; then + download_windows else + # -C -> specifies the output location + # --strip-components -> removes one level of directory nesting curl -L "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz" \ - | tar -zxf - -C deps/zstd --strip-components 1 + | tar -zxf - -C deps/zstd --strip-components 1 fi } @@ -35,4 +44,4 @@ build_zstd() { clean_deps download_zstd -# build_zstd \ No newline at end of file +build_zstd \ No newline at end of file From b308ce4e572ed681122d65ab998e4380acaedd16 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 4 Dec 2024 13:18:29 -0700 Subject: [PATCH 7/9] asdf' --- etc/install-zstd.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/install-zstd.sh b/etc/install-zstd.sh index 7fb4eaa..10936a3 100644 --- a/etc/install-zstd.sh +++ b/etc/install-zstd.sh @@ -24,6 +24,7 @@ download_zstd() { if [ "$is_windows" == "true" ]; then download_windows + exit 0 # no need to build windows else # -C -> specifies the output location # --strip-components -> removes one level of directory nesting From 16e9a9ddf927d1f1bedd8032656b62e75f232b6a Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 4 Dec 2024 13:20:57 -0700 Subject: [PATCH 8/9] Revert "make matrix smaller to save CI" This reverts commit 0f74d7b6123d158823398ef74408c53a14713018. --- .github/workflows/test.yml | 157 ++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 79 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9b4901b..57413b7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,8 +12,7 @@ jobs: strategy: matrix: os: [macos-latest, windows-2019, macos-13] - # node: [16.20.1, 18.x, 20.x, 22.x] - node: [16.20.1] + node: [16.20.1, 18.x, 20.x, 22.x] fail-fast: false runs-on: ${{ matrix.os }} steps: @@ -37,80 +36,80 @@ jobs: shell: bash run: npm test - # container_tests_glibc: - # runs-on: ubuntu-latest - # strategy: - # matrix: - # linux_arch: [s390x, arm64, amd64] - # node: [16.x, 18.x, 20.x, 22.x] - # fail-fast: false - # steps: - # - uses: actions/checkout@v4 - - # - uses: actions/setup-node@v4 - # with: - # node-version: ${{ matrix.node }} - - # - name: Get Full Node.js Version - # id: get_nodejs_version - # shell: bash - # run: | - # echo "version=$(node --print 'process.version.slice(1)')" >> "$GITHUB_OUTPUT" - # echo "ubuntu_version=$(node --print '(+process.version.slice(1).split(`.`).at(0)) > 16 ? `noble` : `bionic`')" >> "$GITHUB_OUTPUT" - - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - - # - name: Run Buildx - # run: | - # docker buildx create --name builder --bootstrap --use - # docker buildx build \ - # --platform linux/${{ matrix.linux_arch }} \ - # --build-arg="NODE_ARCH=${{ matrix.linux_arch == 'amd64' && 'x64' || matrix.linux_arch }}" \ - # --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ - # --build-arg="UBUNTU_VERSION=${{ steps.get_nodejs_version.outputs.ubuntu_version }}" \ - # --build-arg="RUN_TEST=true" \ - # --output type=local,dest=./prebuilds,platform-split=false \ - # -f ./.github/docker/Dockerfile.glibc \ - # . - - # container_tests_musl: - # runs-on: ubuntu-latest - # strategy: - # matrix: - # linux_arch: [amd64, arm64] - # node: [16.20.1, 18.x, 20.x, 22.x] - # fail-fast: false - # steps: - # - uses: actions/checkout@v4 - - # - uses: actions/setup-node@v4 - # with: - # node-version: ${{ matrix.node }} - - # - name: Get Full Node.js Version - # id: get_nodejs_version - # shell: bash - # run: | - # echo "version=$(node --print 'process.version.slice(1)')" >> "$GITHUB_OUTPUT" - - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - - # - name: Run Buildx - # run: | - # docker buildx create --name builder --bootstrap --use - # docker --debug buildx build --progress=plain --no-cache \ - # --platform linux/${{ matrix.linux_arch }} \ - # --build-arg="PLATFORM=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}" \ - # --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ - # --build-arg="RUN_TEST=true" \ - # --output type=local,dest=./prebuilds,platform-split=false \ - # -f ./.github/docker/Dockerfile.musl \ - # . + container_tests_glibc: + runs-on: ubuntu-latest + strategy: + matrix: + linux_arch: [s390x, arm64, amd64] + node: [16.x, 18.x, 20.x, 22.x] + fail-fast: false + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + + - name: Get Full Node.js Version + id: get_nodejs_version + shell: bash + run: | + echo "version=$(node --print 'process.version.slice(1)')" >> "$GITHUB_OUTPUT" + echo "ubuntu_version=$(node --print '(+process.version.slice(1).split(`.`).at(0)) > 16 ? `noble` : `bionic`')" >> "$GITHUB_OUTPUT" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Run Buildx + run: | + docker buildx create --name builder --bootstrap --use + docker buildx build \ + --platform linux/${{ matrix.linux_arch }} \ + --build-arg="NODE_ARCH=${{ matrix.linux_arch == 'amd64' && 'x64' || matrix.linux_arch }}" \ + --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ + --build-arg="UBUNTU_VERSION=${{ steps.get_nodejs_version.outputs.ubuntu_version }}" \ + --build-arg="RUN_TEST=true" \ + --output type=local,dest=./prebuilds,platform-split=false \ + -f ./.github/docker/Dockerfile.glibc \ + . + + container_tests_musl: + runs-on: ubuntu-latest + strategy: + matrix: + linux_arch: [amd64, arm64] + node: [16.20.1, 18.x, 20.x, 22.x] + fail-fast: false + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + + - name: Get Full Node.js Version + id: get_nodejs_version + shell: bash + run: | + echo "version=$(node --print 'process.version.slice(1)')" >> "$GITHUB_OUTPUT" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Run Buildx + run: | + docker buildx create --name builder --bootstrap --use + docker --debug buildx build --progress=plain --no-cache \ + --platform linux/${{ matrix.linux_arch }} \ + --build-arg="PLATFORM=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}" \ + --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ + --build-arg="RUN_TEST=true" \ + --output type=local,dest=./prebuilds,platform-split=false \ + -f ./.github/docker/Dockerfile.musl \ + . From 41d4ce83500ef3c7b1d42c455b250771183dc271 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 4 Dec 2024 13:21:13 -0700 Subject: [PATCH 9/9] final patch? --- .github/workflows/test.yml | 2 +- etc/install-zstd.sh | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 57413b7..762f5b8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: run: npm run install-zstd shell: bash - - name: install dependencies and compmile + - name: install dependencies and compile run: npm install --loglevel verbose shell: bash diff --git a/etc/install-zstd.sh b/etc/install-zstd.sh index 10936a3..852165a 100644 --- a/etc/install-zstd.sh +++ b/etc/install-zstd.sh @@ -26,8 +26,11 @@ download_zstd() { download_windows exit 0 # no need to build windows else - # -C -> specifies the output location - # --strip-components -> removes one level of directory nesting + # tar flags + # -C specifies the output location + # --strip-components removes one level of directory nesting + # curl flags + # -L follows redirects curl -L "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz" \ | tar -zxf - -C deps/zstd --strip-components 1 fi