chore: add memory usage benchmark (#297)

Adds a simple benchmark for making graphs of memory usage during an import

Author: achingbrain

benchmarks/memory/README.md

@@ -0,0 +1,27 @@
+# Memory Benchmark
+
+How much memory does the importer use while importing files?
+
+It should be relatively flat to enable importing files larger than physical memory.
+
+## Usage
+
+```console
+$ npm i
+$ npm start
+
+> [email protected] start
+> npm run build && node dist/src/index.js
+
+
+> [email protected] build
+> aegir build --bundle false
+
+[14:51:28] tsc [started]
+[14:51:33] tsc [completed]
+generating Ed25519 keypair...
+┌─────────┬────────────────┬─────────┬───────────┬──────┐
+│ (index) │ Implementation │  ops/s  │   ms/op   │ runs │
+├─────────┼────────────────┼─────────┼───────────┼──────┤
+//... results here
+```

benchmarks/memory/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "ipfs-unixfs-memory-benchmark",
+  "version": "0.0.0",
+  "description": "Memory benchmarks for ipfs-unixfs-importer",
+  "license": "Apache-2.0 OR MIT",
+  "private": true,
+  "type": "module",
+  "types": "./dist/src/index.d.ts",
+  "files": [
+    "src",
+    "dist",
+    "!dist/test",
+    "!**/*.tsbuildinfo"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/src/index.d.ts",
+      "import": "./dist/src/index.js"
+    }
+  },
+  "eslintConfig": {
+    "extends": "ipfs",
+    "parserOptions": {
+      "sourceType": "module"
+    }
+  },
+  "scripts": {
+    "build": "aegir build --bundle false",
+    "clean": "aegir clean",
+    "lint": "aegir lint",
+    "dep-check": "aegir dep-check",
+    "start": "npm run build && node --expose-gc ./dist/test/index.spec.js"
+  },
+  "devDependencies": {
+    "aegir": "^38.1.2",
+    "blockstore-fs": "^1.0.0",
+    "ipfs-unixfs-importer": "../../packages/ipfs-unixfs-importer",
+    "it-drain": "^2.0.1"
+  }
+}

benchmarks/memory/src/index.ts

@@ -0,0 +1 @@
+export {}

benchmarks/memory/test/index.spec.ts

@@ -0,0 +1,48 @@
+/* eslint-disable no-console */
+
+import { importer } from 'ipfs-unixfs-importer'
+import path from 'node:path'
+import os from 'node:os'
+import fs from 'node:fs'
+import drain from 'it-drain'
+import { FsBlockstore } from 'blockstore-fs'
+
+const ONE_MEG = 1024 * 1024
+
+const FILE_SIZE = ONE_MEG * 1000
+const CHUNK_SIZE = ONE_MEG
+
+async function main (): Promise<void> {
+  const dir = path.join(os.tmpdir(), `test-${Date.now()}`)
+  const blocks = new FsBlockstore(dir)
+  await blocks.open()
+
+  console.info('bytes imported (mb), heap total (mb), heap used (mb), rss (mb)')
+
+  try {
+    await drain(importer([{
+      content: (async function * (): AsyncIterable<Uint8Array> {
+        for (let i = 0; i < FILE_SIZE; i += CHUNK_SIZE) {
+          yield new Uint8Array(CHUNK_SIZE)
+
+          // @ts-expect-error only present when node is run with --expose-gc
+          global.gc()
+
+          console.info(`${i / ONE_MEG}, ${process.memoryUsage().heapTotal / ONE_MEG}, ${process.memoryUsage().heapUsed / ONE_MEG}, ${process.memoryUsage().rss / ONE_MEG}`)
+        }
+      })()
+    }], blocks))
+  } catch (err) {
+    console.error(err)
+  } finally {
+    await blocks.close()
+    fs.rmSync(dir, {
+      recursive: true
+    })
+  }
+}
+
+main().catch(err => {
+  console.error(err) // eslint-disable-line no-console
+  process.exit(1)
+})

benchmarks/memory/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "aegir/src/config/tsconfig.aegir.json",
+  "compilerOptions": {
+    "outDir": "dist"
+  },
+  "include": [
+    "src",
+    "test"
+  ]
+}