feat: added recursive directory walking

Author: aryanjassal

package-lock.json

@@ -35,6 +35,7 @@
     "docs": "shx rm -rf ./docs && typedoc --gitRevision master --tsconfig ./tsconfig.build.json --out ./docs src"
   },
   "dependencies": {
+    "@matrixai/errors": "^1.1.7",
     "@matrixai/logger": "^4.0.3",
     "threads": "^1.7.0",
     "uuid": "^11.0.5"

package.json

@@ -1,5 +1,7 @@
-import type { TarType } from './types';
+import type { TarType, DirectoryContent } from './types';
 import fs from 'fs';
+import path from 'path';
+import * as errors from './errors';
 
 /**
  * The size for each tar block. This is usually 512 bytes.
@@ -15,8 +17,13 @@ function computeChecksum(header: Buffer): number {
 }
 
 function createHeader(filePath: string, stat: fs.Stats, type: TarType): Buffer {
-  const size = type === '0' ? stat.size : 0;
+  if (filePath.length < 1 || filePath.length > 255) {
+    throw new errors.ErrorVirtualTarInvalidFileName(
+      'The file name must be longer than 1 character and shorter than 255 characters',
+    );
+  }
 
+  const size = type === '0' ? stat.size : 0;
   const header = Buffer.alloc(BLOCK_SIZE, 0);
 
   // The TAR headers follow this structure
@@ -41,10 +48,10 @@ function createHeader(filePath: string, stat: fs.Stats, type: TarType): Buffer {
   // 500      12      '\0' (unused)
 
   // FIXME: Assuming file path is under 100 characters long
-  header.write(filePath, 0, 100, 'utf8');
-  // File permissions name will be null
-  // Owner uid will be null
-  // Owner gid will be null
+  header.write(filePath.slice(0, 99).padEnd(100, '\0'), 0, 100, 'utf8');
+  header.write(stat.mode.toString(8).padStart(7, '0') + '\0', 100, 12, 'ascii');
+  header.write(stat.uid.toString(8).padStart(7, '0') + '\0', 108, 12, 'ascii');
+  header.write(stat.gid.toString(8).padStart(7, '0') + '\0', 116, 12, 'ascii');
   header.write(size.toString(8).padStart(7, '0') + '\0', 124, 12, 'ascii');
   // Mtime will be null
   header.write('        ', 148, 8, 'ascii'); // Placeholder for checksum
@@ -56,7 +63,7 @@ function createHeader(filePath: string, stat: fs.Stats, type: TarType): Buffer {
   // Owner group name will be null
   // Device major will be null
   // Device minor will be null
-  // Extended file name will be null
+  header.write(filePath.slice(100).padEnd(155, '\0'), 345, 155, 'utf8');
 
   // Updating with the new checksum
   const checksum = computeChecksum(header);
@@ -86,18 +93,48 @@ async function* readFile(filePath: string): AsyncGenerator<Buffer, void, void> {
   }
 }
 
-// TODO: change path from filepath to a basedir (plus get a fs)
-async function* createTar(filePath: string): AsyncGenerator<Buffer, void, void> {
-  // Create header
-  const stat = await fs.promises.stat(filePath);
-  yield createHeader(filePath, stat, '0');
-  // Get file contents
-  yield *readFile(filePath);
-  // End-of-archive marker
+/**
+ * Traverse a directory recursively and yield file entries.
+ */
+async function* walkDirectory(
+  baseDir: string,
+  relativePath: string = '',
+): AsyncGenerator<DirectoryContent> {
+  const entries = await fs.promises.readdir(path.join(baseDir, relativePath));
+
+  // Sort the entries lexicographically
+  for (const entry of entries.sort()) {
+    const fullPath = path.join(baseDir, relativePath, entry);
+    const stat = await fs.promises.stat(fullPath);
+    const tarPath = path.join(relativePath, entry);
+
+    if (stat.isDirectory()) {
+      yield { path: tarPath + '/', stat: stat, type: '5' };
+      yield* walkDirectory(baseDir, path.join(relativePath, entry));
+    } else if (stat.isFile()) {
+      yield { path: tarPath, stat: stat, type: '0' };
+    }
+  }
+}
+
+async function* createTar(baseDir: string): AsyncGenerator<Buffer, void, void> {
+  for await (const entry of walkDirectory(baseDir)) {
+    // Create header
+    yield createHeader(entry.path, entry.stat, entry.type);
+
+    if (entry.type === '0') {
+      // Get file contents
+      yield* readFile(path.join(baseDir, entry.path));
+    }
+  }
+
+  // End-of-archive marker - two 512-byte null blocks
   yield Buffer.alloc(BLOCK_SIZE, 0);
   yield Buffer.alloc(BLOCK_SIZE, 0);
 }
 
+// NOTE: probably need to remove this, idk
+//  this is a library and should only worry about tarring itself and not writing to fs
 async function writeArchive(inputFile: string, outputFile: string) {
   const fileHandle = await fs.promises.open(outputFile, 'w+');
   for await (const chunk of createTar(inputFile)) {

src/Generator.ts

@@ -0,0 +1,19 @@
+import { AbstractError } from '@matrixai/errors';
+
+class ErrorVirtualTar<T> extends AbstractError<T> {
+  static description = 'VirtualTar errors';
+}
+
+class ErrorVirtualTarUndefinedBehaviour<T> extends ErrorVirtualTar<T> {
+  static description = 'You should never see this error';
+}
+
+class ErrorVirtualTarInvalidFileName<T> extends ErrorVirtualTar<T> {
+  static description = 'The provided file name is invalid';
+}
+
+export {
+  ErrorVirtualTar,
+  ErrorVirtualTarUndefinedBehaviour,
+  ErrorVirtualTarInvalidFileName,
+};

src/errors.ts

@@ -1,5 +1,19 @@
-// 0 = FILE
-// 5 = DIRECTORY
-type TarType = '0' | '5';
+import type { Stats } from 'fs';
 
-export { TarType };
+// FIXME: Using 0s and 5s for files and directories isn't a good way to handle
+//  this. I need to make it simpler so that I can assign and test using strings.
+//  A potential solution is enums, but they don't work with types, so it's a bit
+//  weird.
+type TarFile = '0';
+
+type TarDirectory = '5';
+
+type TarType = TarFile | TarDirectory;
+
+type DirectoryContent = {
+  path: string;
+  stat: Stats;
+  type: TarType;
+};
+
+export type { TarFile, TarDirectory, TarType, DirectoryContent };

src/types.ts

@@ -0,0 +1,7 @@
+import * as errors from './errors';
+
+function never(message: string): never {
+  throw new errors.ErrorVirtualTarUndefinedBehaviour(message);
+}
+
+export { never };

src/utils.ts

@@ -1,12 +1,9 @@
 import { writeArchive } from '@/Generator';
 
 describe('index', () => {
-  test('test', async () => {
+  test.skip('test', async () => {
     await expect(
-      writeArchive(
-        '/home/aryanj/Downloads/tar/FILE.txt',
-        '/home/aryanj/Downloads/tar/FILE.tar',
-      ),
+      writeArchive('/home/aryanj/Downloads', '/home/aryanj/archive.tar'),
     ).toResolve();
-  });
+  }, 60000);
 });