From c942129c4abe9ec8b3e73702382259fb1e44ed96 Mon Sep 17 00:00:00 2001 From: Dorseuil Nicolas Date: Fri, 18 Oct 2024 17:12:39 +0200 Subject: [PATCH 1/4] added new option to install packages --- .../src/build/compileTagCacheProvider.ts | 6 +++ .../build/createImageOptimizationBundle.ts | 31 +++--------- .../open-next/src/build/createMiddleware.ts | 3 ++ .../src/build/createRevalidationBundle.ts | 3 ++ .../open-next/src/build/createServerBundle.ts | 3 ++ .../open-next/src/build/createWarmerBundle.ts | 3 ++ packages/open-next/src/build/installDeps.ts | 50 +++++++++++++++++++ packages/open-next/src/types/open-next.ts | 42 ++++++++++++---- 8 files changed, 108 insertions(+), 33 deletions(-) create mode 100644 packages/open-next/src/build/installDeps.ts diff --git a/packages/open-next/src/build/compileTagCacheProvider.ts b/packages/open-next/src/build/compileTagCacheProvider.ts index 545faea76..f62264342 100644 --- a/packages/open-next/src/build/compileTagCacheProvider.ts +++ b/packages/open-next/src/build/compileTagCacheProvider.ts @@ -2,6 +2,7 @@ import path from "node:path"; import { openNextResolvePlugin } from "../plugins/resolve.js"; import * as buildHelper from "./helper.js"; +import { installDependencies } from "./installDeps.js"; export async function compileTagCacheProvider( options: buildHelper.BuildOptions, @@ -30,4 +31,9 @@ export async function compileTagCacheProvider( }, options, ); + + installDependencies( + providerPath, + options.config.initializationFunction?.install, + ); } diff --git a/packages/open-next/src/build/createImageOptimizationBundle.ts b/packages/open-next/src/build/createImageOptimizationBundle.ts index 66ed29ae5..2dd37dabf 100644 --- a/packages/open-next/src/build/createImageOptimizationBundle.ts +++ b/packages/open-next/src/build/createImageOptimizationBundle.ts @@ -1,4 +1,3 @@ -import cp from "node:child_process"; import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; @@ -7,6 +6,7 @@ import logger from "../logger.js"; import { openNextReplacementPlugin } from "../plugins/replacement.js"; import { openNextResolvePlugin } from "../plugins/resolve.js"; import * as buildHelper from "./helper.js"; +import { installDependencies } from "./installDeps.js"; const require = createRequire(import.meta.url); @@ -108,29 +108,12 @@ export async function createImageOptimizationBundle( // Target should be same as used by Lambda, see https://github.com/sst/sst/blob/ca6f763fdfddd099ce2260202d0ce48c72e211ea/packages/sst/src/constructs/NextjsSite.ts#L114 // For SHARP_IGNORE_GLOBAL_LIBVIPS see: https://github.com/lovell/sharp/blob/main/docs/install.md#aws-lambda - const nodeOutputPath = path.resolve(outputPath); const sharpVersion = process.env.SHARP_VERSION ?? "0.32.6"; - const arch = config.imageOptimization?.arch ?? "arm64"; - const nodeVersion = config.imageOptimization?.nodeVersion ?? "18"; - - //check if we are running in Windows environment then set env variables accordingly. - try { - cp.execSync( - // We might want to change the arch args to cpu args, it seems to be the documented way - `npm install --arch=${arch} --platform=linux --target=${nodeVersion} --libc=glibc --prefix="${nodeOutputPath}" sharp@${sharpVersion}`, - { - stdio: "pipe", - cwd: appPath, - env: { - ...process.env, - SHARP_IGNORE_GLOBAL_LIBVIPS: "1", - }, - }, - ); - } catch (e: any) { - logger.error(e.stdout.toString()); - logger.error(e.stderr.toString()); - logger.error("Failed to install sharp."); - } + installDependencies( + outputPath, + config.imageOptimization?.install ?? { + packages: [`sharp@${sharpVersion}`], + }, + ); } diff --git a/packages/open-next/src/build/createMiddleware.ts b/packages/open-next/src/build/createMiddleware.ts index 7af774d6a..0e90911c1 100644 --- a/packages/open-next/src/build/createMiddleware.ts +++ b/packages/open-next/src/build/createMiddleware.ts @@ -5,6 +5,7 @@ import logger from "../logger.js"; import { type MiddlewareManifest } from "../types/next-types.js"; import { buildEdgeBundle } from "./edge/createEdgeBundle.js"; import * as buildHelper from "./helper.js"; +import { installDependencies } from "./installDeps.js"; export async function createMiddleware(options: buildHelper.BuildOptions) { logger.info(`Bundling middleware function...`); @@ -58,6 +59,8 @@ export async function createMiddleware(options: buildHelper.BuildOptions) { includeCache: config.dangerous?.enableCacheInterception, additionalExternals: config.edgeExternals, }); + + installDependencies(outputPath, config.middleware?.install); } else { await buildEdgeBundle({ entrypoint: path.join( diff --git a/packages/open-next/src/build/createRevalidationBundle.ts b/packages/open-next/src/build/createRevalidationBundle.ts index e3eb59b06..8ba119205 100644 --- a/packages/open-next/src/build/createRevalidationBundle.ts +++ b/packages/open-next/src/build/createRevalidationBundle.ts @@ -4,6 +4,7 @@ import path from "node:path"; import logger from "../logger.js"; import { openNextResolvePlugin } from "../plugins/resolve.js"; import * as buildHelper from "./helper.js"; +import { installDependencies } from "./installDeps.js"; export async function createRevalidationBundle( options: buildHelper.BuildOptions, @@ -41,6 +42,8 @@ export async function createRevalidationBundle( options, ); + installDependencies(outputPath, config.revalidate?.install); + // Copy over .next/prerender-manifest.json file fs.copyFileSync( path.join(appBuildOutputPath, ".next", "prerender-manifest.json"), diff --git a/packages/open-next/src/build/createServerBundle.ts b/packages/open-next/src/build/createServerBundle.ts index b863a2d87..db9c3dce2 100644 --- a/packages/open-next/src/build/createServerBundle.ts +++ b/packages/open-next/src/build/createServerBundle.ts @@ -17,6 +17,7 @@ import { compileCache } from "./compileCache.js"; import { copyTracedFiles } from "./copyTracedFiles.js"; import { generateEdgeBundle } from "./edge/createEdgeBundle.js"; import * as buildHelper from "./helper.js"; +import { installDependencies } from "./installDeps.js"; const require = createRequire(import.meta.url); @@ -267,6 +268,8 @@ async function generateBundle( addMonorepoEntrypoint(outputPath, packagePath); } + installDependencies(outputPath, fnOptions.install); + if (fnOptions.minify) { await minifyServerBundle(outputPath); } diff --git a/packages/open-next/src/build/createWarmerBundle.ts b/packages/open-next/src/build/createWarmerBundle.ts index c765ddfb9..f068a9c60 100644 --- a/packages/open-next/src/build/createWarmerBundle.ts +++ b/packages/open-next/src/build/createWarmerBundle.ts @@ -4,6 +4,7 @@ import path from "node:path"; import logger from "../logger.js"; import { openNextResolvePlugin } from "../plugins/resolve.js"; import * as buildHelper from "./helper.js"; +import { installDependencies } from "./installDeps.js"; export async function createWarmerBundle(options: buildHelper.BuildOptions) { logger.info(`Bundling warmer function...`); @@ -48,4 +49,6 @@ export async function createWarmerBundle(options: buildHelper.BuildOptions) { }, options, ); + + installDependencies(outputPath, config.warmer?.install); } diff --git a/packages/open-next/src/build/installDeps.ts b/packages/open-next/src/build/installDeps.ts new file mode 100644 index 000000000..8c93ab9d7 --- /dev/null +++ b/packages/open-next/src/build/installDeps.ts @@ -0,0 +1,50 @@ +import { execSync } from "child_process"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { InstallOptions } from "types/open-next"; + +import logger from "../logger.js"; + +export function installDependencies( + outputDir: string, + installOptions?: InstallOptions, +) { + try { + if (!installOptions) { + return; + } + const name = outputDir.split("/").pop(); + // First we create a tempDir + const tempInstallDir = fs.mkdtempSync( + path.join(os.tmpdir(), `open-next-install-${name}`), + ); + logger.info(`Installing dependencies for ${name}...`); + // We then need to run install in the tempDir + // We don't install in the output dir directly because it could contain a package.json, and npm would then try to reinstall not complete deps from tracing the files + const installCommand = `npm install --arch=${installOptions.arch ?? "arm64"} --platform=linux --target=${installOptions.nodeVersion ?? "18"} --libc=${installOptions.libc ?? "glibc"} ${installOptions.packages.join(" ")}`; + execSync(installCommand, { + stdio: "pipe", + cwd: tempInstallDir, + env: { + ...process.env, + SHARP_IGNORE_GLOBAL_LIBVIPS: "1", + }, + }); + + // Copy the node_modules to the outputDir + fs.cpSync( + path.join(tempInstallDir, "node_modules"), + path.join(outputDir, "node_modules"), + + { recursive: true, force: true, dereference: true }, + ); + + // Cleanup tempDir + fs.rmSync(tempInstallDir, { recursive: true, force: true }); + logger.info(`Dependencies installed for ${name}`); + } catch (e: any) { + logger.error(e.stdout.toString()); + logger.error("Could not install dependencies"); + } +} diff --git a/packages/open-next/src/types/open-next.ts b/packages/open-next/src/types/open-next.ts index 5efb27de5..e513594e1 100644 --- a/packages/open-next/src/types/open-next.ts +++ b/packages/open-next/src/types/open-next.ts @@ -184,6 +184,31 @@ export interface OverrideOptions extends DefaultOverrideOptions { queue?: IncludedQueue | LazyLoadedOverride; } +export interface InstallOptions { + /** + * List of packages to install + * @example + * ```ts + * install: { + * packages: ["sharp@0.32"] + * } + * ``` + */ + packages: string[]; + /** + * @default "arm64" + */ + arch?: "x64" | "arm64"; + /** + * @default "18" + */ + nodeVersion?: "18" | "20" | "22"; + /** + * @default "glibc" + */ + libc?: "glibc" | "musl"; +} + export interface DefaultFunctionOptions< E extends BaseEventOrResult = InternalEvent, R extends BaseEventOrResult = InternalResult, @@ -202,6 +227,14 @@ export interface DefaultFunctionOptions< * Enable overriding the default lambda. */ override?: DefaultOverrideOptions; + + /** + * Install options for the function. + * This is used to install additional packages to this function. + * For image optimization, it will install sharp by default. + * @default undefined + */ + install?: InstallOptions; } export interface FunctionOptions extends DefaultFunctionOptions { @@ -318,15 +351,6 @@ export interface OpenNextConfig { * @default "s3" */ loader?: IncludedImageLoader | LazyLoadedOverride; - /** - * @default "arm64" - */ - arch?: "x64" | "arm64"; - /** - * @default "18" - */ - - nodeVersion?: "18" | "20"; }; /** From 02a41aa9bb98103d7e3b6fcf2e426e77777b3ec8 Mon Sep 17 00:00:00 2001 From: Dorseuil Nicolas Date: Fri, 18 Oct 2024 18:01:08 +0200 Subject: [PATCH 2/4] fix lint --- .../open-next/src/build/createImageOptimizationBundle.ts | 2 +- packages/open-next/src/build/installDeps.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/open-next/src/build/createImageOptimizationBundle.ts b/packages/open-next/src/build/createImageOptimizationBundle.ts index 2dd37dabf..68165be30 100644 --- a/packages/open-next/src/build/createImageOptimizationBundle.ts +++ b/packages/open-next/src/build/createImageOptimizationBundle.ts @@ -15,7 +15,7 @@ export async function createImageOptimizationBundle( ) { logger.info(`Bundling image optimization function...`); - const { appPath, appBuildOutputPath, config, outputDir } = options; + const { appBuildOutputPath, config, outputDir } = options; // Create output folder const outputPath = path.join(outputDir, "image-optimization-function"); diff --git a/packages/open-next/src/build/installDeps.ts b/packages/open-next/src/build/installDeps.ts index 8c93ab9d7..9c3ec19fa 100644 --- a/packages/open-next/src/build/installDeps.ts +++ b/packages/open-next/src/build/installDeps.ts @@ -22,7 +22,11 @@ export function installDependencies( logger.info(`Installing dependencies for ${name}...`); // We then need to run install in the tempDir // We don't install in the output dir directly because it could contain a package.json, and npm would then try to reinstall not complete deps from tracing the files - const installCommand = `npm install --arch=${installOptions.arch ?? "arm64"} --platform=linux --target=${installOptions.nodeVersion ?? "18"} --libc=${installOptions.libc ?? "glibc"} ${installOptions.packages.join(" ")}`; + const installCommand = `npm install --arch=${ + installOptions.arch ?? "arm64" + } --platform=linux --target=${installOptions.nodeVersion ?? "18"} --libc=${ + installOptions.libc ?? "glibc" + } ${installOptions.packages.join(" ")}`; execSync(installCommand, { stdio: "pipe", cwd: tempInstallDir, From d10f8bc8ec234c19b0169608188641265569eff9 Mon Sep 17 00:00:00 2001 From: conico974 Date: Thu, 24 Oct 2024 19:19:03 +0200 Subject: [PATCH 3/4] review Co-authored-by: Victor Berchet --- packages/open-next/src/build/installDeps.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/open-next/src/build/installDeps.ts b/packages/open-next/src/build/installDeps.ts index 9c3ec19fa..cbbff9be2 100644 --- a/packages/open-next/src/build/installDeps.ts +++ b/packages/open-next/src/build/installDeps.ts @@ -1,7 +1,7 @@ import { execSync } from "child_process"; -import * as fs from "fs"; -import * as os from "os"; -import * as path from "path"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { InstallOptions } from "types/open-next"; import logger from "../logger.js"; From e6a85566eb2a6149a1e9aac3b7b30fa85250b187 Mon Sep 17 00:00:00 2001 From: Dorseuil Nicolas Date: Thu, 24 Oct 2024 19:59:32 +0200 Subject: [PATCH 4/4] changeset & lint fix --- .changeset/chilled-frogs-draw.md | 5 +++++ packages/open-next/src/build/installDeps.ts | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changeset/chilled-frogs-draw.md diff --git a/.changeset/chilled-frogs-draw.md b/.changeset/chilled-frogs-draw.md new file mode 100644 index 000000000..f408d9e0c --- /dev/null +++ b/.changeset/chilled-frogs-draw.md @@ -0,0 +1,5 @@ +--- +"@opennextjs/aws": minor +--- + +Add a new option to install native dependencies on every lambda diff --git a/packages/open-next/src/build/installDeps.ts b/packages/open-next/src/build/installDeps.ts index cbbff9be2..450742155 100644 --- a/packages/open-next/src/build/installDeps.ts +++ b/packages/open-next/src/build/installDeps.ts @@ -1,7 +1,8 @@ -import { execSync } from "child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; + +import { execSync } from "child_process"; import { InstallOptions } from "types/open-next"; import logger from "../logger.js";