feat: handle large zipfiles (#241)

* feat: handle large zipfiles

* chore: changes from review

* chore: close the zipfile

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

Author: ascorbic

plugin/package-lock.json

@@ -42,6 +42,7 @@
   },
   "dependencies": {
     "@netlify/functions": "^0.11.0",
+    "chalk": "^4.1.2",
     "co-body": "^6.1.0",
     "cookie": "^0.4.1",
     "download": "^8.0.0",
@@ -50,7 +51,9 @@
     "linkfs": "^2.1.0",
     "multer": "^1.4.2",
     "node-fetch": "^2.6.1",
+    "node-stream-zip": "^1.15.0",
     "pathe": "^0.2.0",
+    "pretty-bytes": "^5.6.0",
     "tempy": "^1.0.0"
   },
   "devDependencies": {

plugin/package.json

@@ -111,6 +111,7 @@ export function mutateConfig({
       path.posix.join(CACHE_DIR, 'data', '**'),
       path.posix.join(CACHE_DIR, 'query-engine', '**'),
       path.posix.join(CACHE_DIR, 'page-ssr', '**'),
+      '!**/*.js.map',
     ],
     external_node_modules: ['msgpackr-extract'],
     node_bundler: 'esbuild',

plugin/src/helpers/config.ts

@@ -0,0 +1,58 @@
+import process from 'process'
+
+import { yellowBright, redBright } from 'chalk'
+import { stripIndent } from 'common-tags'
+import { existsSync, promises } from 'fs-extra'
+import { async as StreamZip } from 'node-stream-zip'
+import { relative } from 'pathe'
+import prettyBytes from 'pretty-bytes'
+
+// 50MB, which is the documented max, though the hard max seems to be higher
+// eslint-disable-next-line no-magic-numbers
+export const LAMBDA_MAX_SIZE = 1024 * 1024 * 50
+
+// eslint-disable-next-line max-statements
+export const checkZipSize = async (
+  file: string,
+  maxSize: number = LAMBDA_MAX_SIZE,
+): Promise<void> => {
+  if (!existsSync(file)) {
+    console.warn(`Could not check zip size because ${file} does not exist`)
+    return
+  }
+  const fileSize = await promises.stat(file).then(({ size }) => size)
+  if (fileSize < maxSize) {
+    return
+  }
+  // We don't fail the build, because the actual hard max size is larger so it might still succeed
+  console.log(
+    redBright(stripIndent`
+      The function zip ${yellowBright(
+        relative(process.cwd(), file),
+      )} size is ${prettyBytes(
+      fileSize,
+    )}, which is larger than the maximum supported size of ${prettyBytes(
+      maxSize,
+    )}.
+      There are a few reasons this could happen, such as accidentally bundling a large dependency or adding lots of files to "included_files".
+    `),
+  )
+  const zip = new StreamZip({ file })
+  console.log(`Contains ${await zip.entriesCount} files`)
+  const sortedFiles = Object.values(await zip.entries()).sort(
+    (entryA, entryB) => entryB.size - entryA.size,
+  )
+
+  const largest = {}
+  const MAX_ROWS = 10
+  for (let idx = 0; idx < MAX_ROWS && idx < sortedFiles.length; idx++) {
+    largest[`${idx + 1}`] = {
+      File: sortedFiles[idx].name,
+      'Compressed Size': prettyBytes(sortedFiles[idx].compressedSize),
+      'Uncompressed Size': prettyBytes(sortedFiles[idx].size),
+    }
+  }
+  await zip.close()
+  console.log(yellowBright`\n\nThese are the largest files in the zip:`)
+  console.table(largest)
+}

plugin/src/helpers/verification.ts

@@ -6,6 +6,7 @@ import fs from 'fs-extra'
 import { normalizedCacheDir, restoreCache, saveCache } from './helpers/cache'
 import { checkGatsbyConfig, mutateConfig, spliceConfig } from './helpers/config'
 import { writeFunctions } from './helpers/functions'
+import { checkZipSize } from './helpers/verification'
 
 // eslint-disable-next-line no-template-curly-in-string
 const lmdbCacheString = 'process.cwd(), `.cache/${cacheDbFile}`'
@@ -104,8 +105,11 @@ The plugin no longer uses this and it should be deleted to avoid conflicts.\n`)
 }
 
 export async function onPostBuild({
-  constants: { PUBLISH_DIR },
+  constants: { PUBLISH_DIR, FUNCTIONS_DIST },
   utils,
 }): Promise<void> {
   await saveCache({ publish: PUBLISH_DIR, utils })
+  for (const func of ['api', 'dsg', 'ssr']) {
+    await checkZipSize(path.join(FUNCTIONS_DIST, `__${func}.zip`))
+  }
 }