perf: replace archiver and tar-fs with modern-tar

Author: ayuhito

package-lock.json

@@ -1,6 +1,6 @@
 import { copyFile } from "fs/promises";
 import path from "path";
-import tar from "tar-fs";
+import { pipeline } from "stream/promises";
 import {
   AbstractStartedContainer,
   AbstractStoppedContainer,
@@ -128,19 +128,12 @@ export class StoppedSeleniumRecordingContainer extends StoppedSeleniumContainer
 
     log.debug("Unpacking archive...", { containerId: ffmpegContainerId });
     const destinationDir = tmp.dirSync({ keep: false });
-    await this.extractTarStreamToDest(archiveStream, destinationDir.name);
+    const { unpackTar } = await import("modern-tar/fs");
+    await pipeline(archiveStream, unpackTar(destinationDir.name));
     log.debug("Unpacked archive", { containerId: ffmpegContainerId });
 
     const videoFile = path.resolve(destinationDir.name, "video.mp4");
     await copyFile(videoFile, target);
     log.debug(`Extracted video to "${target}"`, { containerId: ffmpegContainerId });
   }
-
-  private async extractTarStreamToDest(tarStream: NodeJS.ReadableStream, dest: string): Promise<void> {
-    await new Promise<void>((resolve) => {
-      const destination = tar.extract(dest);
-      tarStream.pipe(destination);
-      destination.on("finish", resolve);
-    });
-  }
 }

packages/modules/selenium/src/selenium-container.ts

@@ -32,28 +32,25 @@
   "dependencies": {
     "@balena/dockerignore": "^1.0.2",
     "@types/dockerode": "^3.3.44",
-    "archiver": "^7.0.1",
     "async-lock": "^1.4.1",
     "byline": "^5.0.0",
     "debug": "^4.4.3",
     "docker-compose": "^1.3.0",
     "dockerode": "^4.0.8",
     "get-port": "^7.1.0",
+    "modern-tar": "^0.5.3",
     "proper-lockfile": "^4.1.2",
     "properties-reader": "^2.3.0",
     "ssh-remote-port-forward": "^1.0.4",
-    "tar-fs": "^3.1.1",
     "tmp": "^0.2.5",
     "undici": "^7.16.0"
   },
   "devDependencies": {
-    "@types/archiver": "^6.0.3",
     "@types/async-lock": "^1.4.2",
     "@types/byline": "^4.2.36",
     "@types/debug": "^4.1.12",
     "@types/proper-lockfile": "^4.1.4",
     "@types/properties-reader": "^2.1.3",
-    "@types/tar-fs": "^2.0.4",
     "@types/tmp": "^0.2.6"
   }
 }

packages/testcontainers/package.json

@@ -4,7 +4,6 @@ import byline from "byline";
 import Dockerode, { ImageBuildOptions, ImageInspectInfo } from "dockerode";
 import { existsSync, promises as fs } from "fs";
 import path from "path";
-import tar from "tar-fs";
 import { buildLog, log, pullLog } from "../../../common";
 import { getAuthConfig } from "../../auth/get-auth-config";
 import { ImageName } from "../../image-name";
@@ -23,14 +22,14 @@ export class DockerImageClient implements ImageClient {
     try {
       log.debug(`Building image "${opts.t}" with context "${context}"...`);
       const isDockerIgnored = await this.createIsDockerIgnoredFunction(context);
-      const tarStream = tar.pack(context, {
-        ignore: (aPath) => {
-          const relativePath = path.relative(context, aPath);
-          if (relativePath === opts.dockerfile) {
-            return false;
-          } else {
-            return isDockerIgnored(relativePath);
+      const { packTar } = await import("modern-tar/fs");
+      const tarStream = packTar(context, {
+        filter: (path) => {
+          if (path === opts.dockerfile) {
+            return true;
           }
+
+          return !isDockerIgnored(path);
         },
       });
       await new Promise<void>((resolve) => {

packages/testcontainers/src/container-runtime/clients/image/docker-image-client.ts

@@ -411,7 +411,11 @@ describe("GenericContainer", { timeout: 180_000 }, () => {
       .withExposedPorts(8080)
       .start();
 
-    expect((await container.exec(`stat -c "%a %n" /tmp/newdir/test.txt`)).output).toContain("777");
+    // Check directory permissions
+    expect((await container.exec(`stat -c "%a %n" /tmp/newdir`)).output).toContain("777");
+
+    // Verify files retain their original permissions
+    expect((await container.exec(`stat -c "%a %n" /tmp/newdir/test.txt`)).output).toContain("644");
   });
 
   it("should copy directory to started container", async () => {

packages/testcontainers/src/generic-container/generic-container.test.ts

@@ -1,7 +1,7 @@
-import archiver from "archiver";
 import AsyncLock from "async-lock";
 import { Container, ContainerCreateOptions, HostConfig } from "dockerode";
 import { Readable } from "stream";
+import { buffer } from "stream/consumers";
 import { containerLog, hash, log, toNanos } from "../common";
 import { ContainerRuntimeClient, getContainerRuntimeClient, ImageName } from "../container-runtime";
 import { CONTAINER_STATUSES } from "../container-runtime/clients/container/types";
@@ -178,9 +178,30 @@ export class GenericContainer implements TestContainer {
       await client.container.connectToNetwork(container, network, this.networkAliases);
     }
 
-    if (this.filesToCopy.length > 0 || this.directoriesToCopy.length > 0 || this.contentsToCopy.length > 0) {
-      const archive = this.createArchiveToCopyToContainer();
-      archive.finalize();
+    const sources = [
+      ...this.filesToCopy.map((file) => ({
+        type: "file" as const,
+        source: file.source,
+        target: file.target,
+        mode: file.mode,
+      })),
+      ...this.directoriesToCopy.map((dir) => ({
+        type: "directory" as const,
+        source: dir.source,
+        target: dir.target,
+        mode: dir.mode,
+      })),
+      ...(await Promise.all(
+        this.contentsToCopy.map(async ({ content, target, mode }) => {
+          const data = content instanceof Readable ? await buffer(content) : content;
+          return { type: "content" as const, content: data, target, mode };
+        })
+      )),
+    ];
+
+    if (sources.length > 0) {
+      const { packTar } = await import("modern-tar/fs");
+      const archive = packTar(sources);
       await client.container.putArchive(container, archive, "/");
     }
 
@@ -255,22 +276,6 @@ export class GenericContainer implements TestContainer {
     }
   }
 
-  private createArchiveToCopyToContainer(): archiver.Archiver {
-    const tar = archiver("tar");
-
-    for (const { source, target, mode } of this.filesToCopy) {
-      tar.file(source, { name: target, mode });
-    }
-    for (const { source, target, mode } of this.directoriesToCopy) {
-      tar.directory(source, target, { mode });
-    }
-    for (const { content, target, mode } of this.contentsToCopy) {
-      tar.append(content, { name: target, mode });
-    }
-
-    return tar;
-  }
-
   protected containerStarted?(
     container: StartedTestContainer,
     inspectResult: InspectResult,

packages/testcontainers/src/generic-container/generic-container.ts

@@ -1,7 +1,7 @@
-import archiver from "archiver";
 import AsyncLock from "async-lock";
 import Dockerode, { ContainerInspectInfo } from "dockerode";
 import { Readable } from "stream";
+import { buffer } from "stream/consumers";
 import { containerLog, log } from "../common";
 import { ContainerRuntimeClient, getContainerRuntimeClient } from "../container-runtime";
 import { getReaper } from "../reaper/reaper";
@@ -181,31 +181,64 @@ export class StartedGenericContainer implements StartedTestContainer {
 
   public async copyFilesToContainer(filesToCopy: FileToCopy[]): Promise<void> {
     log.debug(`Copying files to container...`, { containerId: this.container.id });
+    if (filesToCopy.length === 0) {
+      return;
+    }
+
     const client = await getContainerRuntimeClient();
-    const tar = archiver("tar");
-    filesToCopy.forEach(({ source, target }) => tar.file(source, { name: target }));
-    tar.finalize();
-    await client.container.putArchive(this.container, tar, "/");
+    const { packTar } = await import("modern-tar/fs");
+
+    const sources = filesToCopy.map((file) => ({
+      type: "file" as const,
+      source: file.source,
+      target: file.target,
+      mode: file.mode,
+    }));
+    const archive = packTar(sources);
+
+    await client.container.putArchive(this.container, archive, "/");
     log.debug(`Copied files to container`, { containerId: this.container.id });
   }
 
   public async copyDirectoriesToContainer(directoriesToCopy: DirectoryToCopy[]): Promise<void> {
     log.debug(`Copying directories to container...`, { containerId: this.container.id });
+    if (directoriesToCopy.length === 0) {
+      return;
+    }
+
     const client = await getContainerRuntimeClient();
-    const tar = archiver("tar");
-    directoriesToCopy.forEach(({ source, target }) => tar.directory(source, target));
-    tar.finalize();
-    await client.container.putArchive(this.container, tar, "/");
+    const { packTar } = await import("modern-tar/fs");
+
+    const sources = directoriesToCopy.map((dir) => ({
+      type: "directory" as const,
+      source: dir.source,
+      target: dir.target,
+      mode: dir.mode,
+    }));
+    const archive = packTar(sources);
+
+    await client.container.putArchive(this.container, archive, "/");
     log.debug(`Copied directories to container`, { containerId: this.container.id });
   }
 
   public async copyContentToContainer(contentsToCopy: ContentToCopy[]): Promise<void> {
     log.debug(`Copying content to container...`, { containerId: this.container.id });
+    if (contentsToCopy.length === 0) {
+      return;
+    }
+
+    const sources = await Promise.all(
+      contentsToCopy.map(async ({ content, target, mode }) => {
+        const processedContent = content instanceof Readable ? await buffer(content) : content;
+        return { type: "content" as const, content: processedContent, target, mode };
+      })
+    );
+
     const client = await getContainerRuntimeClient();
-    const tar = archiver("tar");
-    contentsToCopy.forEach(({ content, target, mode }) => tar.append(content, { name: target, mode: mode }));
-    tar.finalize();
-    await client.container.putArchive(this.container, tar, "/");
+    const { packTar } = await import("modern-tar/fs");
+    const archive = packTar(sources);
+
+    await client.container.putArchive(this.container, archive, "/");
     log.debug(`Copied content to container`, { containerId: this.container.id });
   }
 

packages/testcontainers/src/generic-container/started-generic-container.ts

@@ -6,11 +6,11 @@
     "types": [
       "vitest/globals"
     ],
-    "module": "commonjs",
+    "module": "Node16",
     "declaration": true,
     "sourceMap": true,
     "strict": true,
-    "moduleResolution": "node",
+    "moduleResolution": "node16",
     "esModuleInterop": true,
     "skipLibCheck": true,
     "forceConsistentCasingInFileNames": true,