feat: support gcloudignore (#65)

bharathkkb · web-flow · commit bc4a13eb460a · 2021-06-09T11:28:31.000-05:00
* feat: support gcloudignore

* fixlint

* use posix path
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -34,10 +34,12 @@
     "@actions/core": "^1.2.6",
     "@types/archiver": "^3.1.0",
     "archiver": "^5.0.0",
+    "fast-glob": "^3.2.5",
     "fs": "0.0.1-security",
     "gaxios": "^3.1.0",
     "google-auth-library": "^6.0.6",
     "googleapis": "^58.0.0",
+    "ignore": "^4.0.6",
     "yaml": "^1.10.0"
   },
   "devDependencies": {
@@ -54,6 +56,7 @@
     "eslint-config-prettier": "^6.11.0",
     "eslint-plugin-prettier": "^3.1.4",
     "mocha": "^8.1.1",
+    "node-stream-zip": "^1.13.4",
     "prettier": "^2.0.5",
     "ts-node": "^8.10.2",
     "typescript": "^3.9.7"
diff --git a/src/util.ts b/src/util.ts
@@ -18,6 +18,9 @@ import * as fs from 'fs';
 import * as core from '@actions/core';
 import { Gaxios } from 'gaxios';
 import * as Archiver from 'archiver';
+import * as path from 'path';
+import ignore from 'ignore';
+import fg from 'fast-glob';
 
 /**
  * Zip a directory.
@@ -56,12 +59,44 @@ export async function zipDir(
     });
     archive.pipe(output);
     // Add dir to root of archive
-    archive.directory(dirPath, false);
+    getFiles(dirPath).forEach((filepath) => {
+      archive.glob(filepath, {
+        cwd: dirPath,
+        noglobstar: true,
+      });
+    });
     // Finish writing files
     archive.finalize();
   });
 }
 
+/**
+ * @param dir dir to collect files from
+ * @returns list of files that are not ignored
+ */
+export function getFiles(dir: string): string[] {
+  const files = fg.sync(['**'], { cwd: dir });
+  // return list of files that are not ignored
+  return ignore().add(getGcloudIgnores(dir)).filter(files);
+}
+
+/**
+ * @param dir dir which may contain .gcloudignore file
+ * @returns list of ignores in .gcloudignore if present
+ */
+export function getGcloudIgnores(dir: string): string[] {
+  const gcloudIgnorePath = path.posix.join(dir, '.gcloudignore');
+  if (!fs.existsSync(gcloudIgnorePath)) {
+    return [];
+  }
+  // read .gcloudignore, split on newline
+  return fs
+    .readFileSync(gcloudIgnorePath, { encoding: 'utf-8' })
+    .toString()
+    .split(/\r?\n/)
+    .map((s) => s.trim());
+}
+
 /**
  * Deletes a zip file from disk.
  *
diff --git a/tests/test-func-ignore-node/.gcloudignore b/tests/test-func-ignore-node/.gcloudignore
@@ -0,0 +1 @@
+node_modules/
diff --git a/tests/test-func-ignore-node/foo/data.txt b/tests/test-func-ignore-node/foo/data.txt
@@ -0,0 +1 @@
+data
diff --git a/tests/test-func-ignore-node/index.js b/tests/test-func-ignore-node/index.js
@@ -0,0 +1,10 @@
+/**
+ * Responds to any HTTP request.
+ *
+ * @param {!express:Request} req HTTP request context.
+ * @param {!express:Response} res HTTP response context.
+ */
+exports.helloWorld = (req, res) => {
+  let message = req.query.message || req.body.message || 'Hello World!!';
+  res.status(200).send(message);
+};
diff --git a/tests/test-func-ignore-node/notIgnored.txt b/tests/test-func-ignore-node/notIgnored.txt
@@ -0,0 +1 @@
+baz
diff --git a/tests/test-func-ignore-node/package.json b/tests/test-func-ignore-node/package.json
@@ -0,0 +1,4 @@
+{
+  "name": "sample-http",
+  "version": "0.0.1"
+}
diff --git a/tests/test-func-ignore/.gcloudignore b/tests/test-func-ignore/.gcloudignore
@@ -0,0 +1 @@
+*.txt
diff --git a/tests/test-func-ignore/ignore.txt b/tests/test-func-ignore/ignore.txt
@@ -0,0 +1 @@
+foo
diff --git a/tests/test-func-ignore/index.js b/tests/test-func-ignore/index.js
@@ -0,0 +1,10 @@
+/**
+ * Responds to any HTTP request.
+ *
+ * @param {!express:Request} req HTTP request context.
+ * @param {!express:Response} res HTTP response context.
+ */
+exports.helloWorld = (req, res) => {
+  let message = req.query.message || req.body.message || 'Hello World!!';
+  res.status(200).send(message);
+};
diff --git a/tests/test-func-ignore/package.json b/tests/test-func-ignore/package.json
@@ -0,0 +1,4 @@
+{
+  "name": "sample-http",
+  "version": "0.0.1"
+}
diff --git a/tests/util.test.ts b/tests/util.test.ts
@@ -0,0 +1,94 @@
+import { expect } from 'chai';
+import os from 'os';
+import * as fs from 'fs';
+import 'mocha';
+import * as path from 'path';
+import { zipDir } from '../src/util';
+import StreamZip from 'node-stream-zip';
+
+const testDirNoIgnore = 'tests/test-node-func';
+const testDirSimpleIgnore = 'tests/test-func-ignore';
+const testDirNodeIgnore = 'tests/test-func-ignore-node';
+const name = `zip-${Math.round(Math.random() * 100000)}`;
+describe('Zip', function () {
+  it('creates a zipfile with correct files without gcloudignore', async function () {
+    const zf = await zipDir(
+      testDirNoIgnore,
+      path.posix.join(os.tmpdir(), name),
+    );
+    const uzf = new StreamZip.async({ file: zf });
+    const filesInsideZip = await uzf.entries();
+    const expectedFiles = getNonIgnoredFiles(testDirNoIgnore, testDirNoIgnore);
+    expect(await uzf.entriesCount).equal(expectedFiles.length);
+    for (const entry of Object.values(filesInsideZip)) {
+      expect(expectedFiles).to.be.include(entry.name);
+    }
+  });
+
+  it('creates a zipfile with correct files with simple gcloudignore', async function () {
+    const zf = await zipDir(
+      testDirSimpleIgnore,
+      path.posix.join(os.tmpdir(), name),
+    );
+    const uzf = new StreamZip.async({ file: zf });
+    const filesInsideZip = await uzf.entries();
+    const expectedFiles = getNonIgnoredFiles(
+      testDirSimpleIgnore,
+      testDirSimpleIgnore,
+      new Set(['ignore.txt', '.gcloudignore']),
+    );
+    expect(await uzf.entriesCount).equal(expectedFiles.length);
+    for (const entry of Object.values(filesInsideZip)) {
+      expect(expectedFiles).to.be.include(entry.name);
+    }
+  });
+
+  it('creates a zipfile with correct files with dir gcloudignore', async function () {
+    const zf = await zipDir(
+      testDirNodeIgnore,
+      path.posix.join(os.tmpdir(), name),
+    );
+    const uzf = new StreamZip.async({ file: zf });
+    const filesInsideZip = await uzf.entries();
+    const expectedFiles = getNonIgnoredFiles(
+      testDirNodeIgnore,
+      testDirNodeIgnore,
+      new Set([
+        'node_modules/foo/foo.txt',
+        'node_modules/bar/bar.txt',
+        '.gcloudignore',
+      ]),
+    );
+    expect(await uzf.entriesCount).equal(expectedFiles.length);
+    for (const entry of Object.values(filesInsideZip)) {
+      expect(expectedFiles).to.be.include(entry.name);
+    }
+  });
+});
+
+function getNonIgnoredFiles(
+  parentDir: string,
+  directory: string,
+  ignore: Set<string> = new Set(),
+  fileList: string[] = [],
+): string[] {
+  const items = fs.readdirSync(directory);
+  for (const item of items) {
+    const stat = fs.statSync(path.posix.join(directory, item));
+    if (stat.isDirectory())
+      fileList = getNonIgnoredFiles(
+        parentDir,
+        path.posix.join(directory, item),
+        ignore,
+        fileList,
+      );
+    else {
+      const fPath = path.posix.relative(
+        parentDir,
+        path.posix.join(directory, item),
+      );
+      if (!ignore.has(fPath)) fileList.push(fPath);
+    }
+  }
+  return fileList;
+}

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +{
 +  "name": "sample-http",
 +  "version": "0.0.1"
 +}