remove zstd-napi (switch to native nodejs)

Author: williamstein

src/packages/backend/conat/persist.ts

@@ -11,15 +11,15 @@ additional persist server:
 import "./index";
 import betterSqlite3 from "better-sqlite3";
 import { initContext } from "@cocalc/conat/persist/context";
-import { compress, decompress } from "zstd-napi";
+const { zstdCompressSync, zstdDecompressSync } = require("node:zlib");
 import { syncFiles } from "@cocalc/backend/data";
 import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists";
 import { statSync, copyFileSync } from "node:fs";
 
 initContext({
   betterSqlite3,
-  compress,
-  decompress,
+  compress: zstdCompressSync,
+  decompress: zstdDecompressSync,
   syncFiles,
   ensureContainingDirectoryExists,
   statSync,

src/packages/backend/package.json

@@ -63,8 +63,7 @@
     "reflect-sync": "^0.14.4",
     "rimraf": "^5.0.5",
     "shell-escape": "^0.2.0",
-    "tmp-promise": "^3.0.3",
-    "zstd-napi": "^0.0.12"
+    "tmp-promise": "^3.0.3"
   },
   "repository": {
     "type": "git",

src/packages/conat/persist/storage.ts

@@ -21,8 +21,7 @@ less than 1 read/write per second for each.  Thus memory is critical, and
 supporting at least 1000 writes/second is what we need.
 Fortunately, this implementation can do ~50,000+ writes per second and read
 over 500,000 per second.  Yes, it blocks the main thread, but by using
-better-sqlite3 and zstd-napi, we get 10x speed increases over async code,
-so this is worth it.
+better-sqlite3, we get speed increases over async code, so this is worth it.
 
 
 COMPRESSION:
@@ -36,6 +35,7 @@ https://github.com/antoniomuso/lz4-napi/issues/678
 I also tried the rust sync snappy and it had a similar memory leak.  Finally,
 I tried zstd-napi and it has a very fast sync implementation that does *not*
 need async pauses to not leak memory. So zstd-napi it is.
+I then switched to the zstd built into nodejs.
 And I like zstandard anyways.
 
 TIERED STORAGE:

src/packages/lite/package.json

@@ -5,18 +5,16 @@
   "exports": {
     "./*": "./dist/*.js"
   },
-  "keywords": [
-    "lite",
-    "cocalc"
-  ],
+  "keywords": ["lite", "cocalc"],
   "scripts": {
     "preinstall": "npx only-allow pnpm",
     "clean": "rm -rf build dist node_modules sea/sea-prep.blob",
     "build": "pnpm exec tsc --build",
     "tsc": "pnpm exec tsc --watch  --pretty --preserveWatchOutput",
     "test": "exit 0",
     "depcheck": "pnpx depcheck --ignores @types/debug,@types/jest,@types/node,@vercel/ncc",
-    "build:bundle": "cd sea && ./build-bundle.sh",
+    "build:static": "cd ../static && pnpm clean && pnpm install && pnpm build",
+    "build:bundle": "pnpm build:static && cd sea && ./build-bundle.sh",
     "build:tarball": "pnpm build:bundle && cd build && tar Jcvf bundle.tar.xz bundle",
     "run:bundle": "node ./build/bundle/bundle/index.js",
     "sea": "cd sea && ./build-sea.sh",

src/packages/package.json

@@ -34,10 +34,6 @@
       "axios@<1.12.0": "^1.12.0",
       "tmp@<0.2.4": "^0.2.4"
     },
-    "onlyBuiltDependencies": [
-      "better-sqlite3",
-      "node-pty",
-      "zstd-napi"
-    ]
+    "onlyBuiltDependencies": ["better-sqlite3", "node-pty"]
   }
 }

src/packages/pnpm-lock.yaml

@@ -6,4 +6,3 @@ packages:
   - "!**/cdn/dist/**"
 onlyBuiltDependencies:
   - better-sqlite3
-  - zstd-napi