feat: zip reader file path glob filter and helpers (VF-000) (#77)

<!-- You can erase any parts of this template not applicable to your Pull Request. -->

**Fixes or implements VF-000**

### Brief description. What is this change?

- Adds a glob filter to file reader
- Adds helper methods to iterator items to access file contents

Co-authored-by: Tyler Stewart <[email protected]>

Author: trs

package.json

@@ -22,6 +22,7 @@
     "jszip": "3.7.1",
     "lodash": "^4.17.11",
     "luxon": "^1.21.3",
+    "minimatch": "5.0.1",
     "rate-limiter-flexible": "^2.2.2",
     "sinon": "^10.0.0"
   },

src/common/zip/reader.ts

@@ -1,4 +1,6 @@
+import { AnyRecord } from '@voiceflow/common';
 import JSZip from 'jszip';
+import minimatch from 'minimatch';
 
 import { isZipFile } from './guard';
 
@@ -34,6 +36,12 @@ export interface ZipReaderConfig {
 export interface ZipEntry {
   name: string;
   content: Uint8Array;
+  toString: (decoder?: TextDecoder) => string;
+  toJSON: <T = AnyRecord>(decoder?: TextDecoder) => T;
+}
+
+export interface ZipGetFilesOptions {
+  path: string;
 }
 
 export class ZipReader {
@@ -48,12 +56,15 @@ export class ZipReader {
     };
   }
 
-  public async *getFiles(): AsyncGenerator<ZipEntry> {
-    yield* this.getFilesRecursively(this.zip, this.config.maxZipRecursionDepth);
+  public async *getFiles(options?: Partial<ZipGetFilesOptions>): AsyncGenerator<ZipEntry> {
+    yield* this.getFilesRecursively(this.zip, {
+      path: '**/*',
+      ...(options ?? {}),
+    });
   }
 
-  private async *getFilesRecursively(zip: JSZip, maxDepth: number, currentDepth = 0): AsyncGenerator<ZipEntry> {
-    const objects = zip.filter(() => true);
+  private async *getFilesRecursively(zip: JSZip, config: ZipGetFilesOptions, currentDepth = 0): AsyncGenerator<ZipEntry> {
+    const objects = zip.filter((_, file) => minimatch(file.name, config.path));
 
     let fileCount = 0;
     let totalSize = 0;
@@ -75,14 +86,16 @@ export class ZipReader {
         throw new Error(`Total file size exceeded maximum (${this.config.maxUnzipSizeBytes} bytes)`);
       }
 
-      if (currentDepth < maxDepth && isZipFile(content)) {
+      if (currentDepth < this.config.maxZipRecursionDepth && isZipFile(content)) {
         // eslint-disable-next-line no-await-in-loop
         const zip = await JSZip.loadAsync(content);
-        yield* this.getFilesRecursively(zip, maxDepth, currentDepth + 1);
+        yield* this.getFilesRecursively(zip, config, currentDepth + 1);
       } else {
         yield {
           name: obj.name,
           content,
+          toString: (decoder = new TextDecoder()) => decoder.decode(content),
+          toJSON: <T = AnyRecord>(decoder = new TextDecoder()) => JSON.parse(decoder.decode(content)) as T,
         };
       }
     }

tests/common/zip/reader.unit.ts

@@ -19,6 +19,7 @@ describe('ZipReader', () => {
     zip = new JSZip();
     zip.file('file.json', JSON.stringify({ hello: 'world' }));
     zip.file('nested/file.zip', new JSZip().file('inside.json', JSON.stringify({ a: 1 })).generateNodeStream());
+    zip.file('nested/other.json', JSON.stringify({ key: true }));
   });
 
   afterEach(() => {
@@ -29,12 +30,16 @@ describe('ZipReader', () => {
     const reader = new ZipReader(zip);
     const files = await asyncArrayFrom(reader.getFiles());
 
-    expect(files.length).to.equal(2);
+    expect(files.length).to.equal(3);
+
     expect(files[0].name).to.equal('file.json');
     expect(new TextDecoder().decode(files[0].content)).to.equal(JSON.stringify({ hello: 'world' }));
 
     expect(files[1].name).to.equal('inside.json');
     expect(new TextDecoder().decode(files[1].content)).to.equal(JSON.stringify({ a: 1 }));
+
+    expect(files[2].name).to.equal('nested/other.json');
+    expect(new TextDecoder().decode(files[2].content)).to.equal(JSON.stringify({ key: true }));
   });
 
   it('throws if total size exceeds limit', async () => {
@@ -55,8 +60,17 @@ describe('ZipReader', () => {
     const reader = new ZipReader(zip, { maxZipRecursionDepth: 0 });
     const files = await asyncArrayFrom(reader.getFiles());
 
-    expect(files.length).to.equal(2);
+    expect(files.length).to.equal(3);
     expect(files[0].name).to.equal('file.json');
     expect(files[1].name).to.equal('nested/file.zip');
+    expect(files[2].name).to.equal('nested/other.json');
+  });
+
+  it('filters by path glob', async () => {
+    const reader = new ZipReader(zip);
+    const files = await asyncArrayFrom(reader.getFiles({ path: 'nested/*.json' }));
+
+    expect(files.length).to.equal(1);
+    expect(files[0].name).to.equal('nested/other.json');
   });
 });

tsconfig.json

@@ -1,7 +1,8 @@
 {
   "extends": "@voiceflow/tsconfig",
   "compilerOptions": {
-    "baseUrl": "./src"
+    "baseUrl": "./src",
+    "lib": ["DOM", "ES2020"]
   },
   "compileOnSave": false,
   "include": [

yarn.lock

@@ -1782,6 +1782,13 @@ brace-expansion@^1.1.7:
     balanced-match "^1.0.0"
     concat-map "0.0.1"
 
+brace-expansion@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae"
+  integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==
+  dependencies:
+    balanced-match "^1.0.0"
+
 braces@^2.3.1:
   version "2.3.2"
   resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.2.tgz#5979fd3f14cd531565e5fa2df1abfff1dfaee729"
@@ -5212,6 +5219,13 @@ [email protected], minimatch@^3.0.4:
   dependencies:
     brace-expansion "^1.1.7"
 
+[email protected]:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.0.1.tgz#fb9022f7528125187c92bd9e9b6366be1cf3415b"
+  integrity sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==
+  dependencies:
+    brace-expansion "^2.0.1"
+
 [email protected]:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619"