feat(build,config)!: rework extension development workflow (#6571)

This changeset makes a light breaking change to the way local extension build plugins work. This fixes an issue where functions injected using the `functions.generate` API silently fail to include `node_modules` required for the function to work.

Currently, during `netlify build` and `netlify dev`, build will automatically build your extension and include any build plugins produced by that extension in the build process if you specify this configuration in your `netlify.toml` (documented [here](https://developers.netlify.com/sdk/build-event-handlers/debug-and-test/)):

```toml
# netlify.toml

[[integrations]]
name = "my-local-extension"

[[integrations.dev]]
path = "../some-local-path-to-integration"
force_run_in_build = true
```

To make it short(ish): This process makes a lot of assumptions about the default file structure of a Netlify Extension. It also installs dev-mode extension build plugins using a different code path than for production extension build plugins: In production, we do an `npm install` against a tarball; in development, we point build at a directory and no installation is performed. (This is the crux of the `functions.generate` problem: the directory we point build at is a temporary build directory and does not contain `node_modules`.) We use dev-mode extensions in SDK/build integration tests, so this problem breaks our existing tests and generally makes it difficult to trust that SDK tests will catch real-world, user-facing build plugin problems.

With these changes, build no longer automatically builds extensions. Instead, it assumes that you've already built your extension, and will now install extension build plugins from a local tarball. This makes dev mode behave very similarly to production mode builds.

You can now point build at a tarball:

```toml
# netlify.toml

[[integrations]]
name = "my-local-extension"

[[integrations.dev]]
path = "../some-local-path-to-integration/buildhooks.tgz"
# Note: force_run_in_build is no longer required
```
To make the transition on this change easier, you can instead point build at a directory, and the historical default extension build hooks tarball path will be inferred:

```toml
# netlify.toml

[[integrations]]
name = "my-local-extension"

[[integrations.dev]]
# Expands to "../some-local-path-to-integration/.ntli/site/static/packages/buildhooks.tgz"
path = "../some-local-path-to-integration"
```

There's part of me that thinks, "This behaves slightly differently than local build plugins, and that asymmetry sucks." But: Local build plugins are meant to work when developed in the same tree as the site they're modifying. For better or for worse, developing extensions in the same tree as a site is not supported. So, this asymmetry actually feels appropriate to me.

---

Footnote: This corner of the code had a lot of dead code, code that was untyped, code that was added to avoid updating test fixtures, code that was unnecessarily complex, etc. etc. I did some refactoring along the way to make this functionality easier to implement, and those refactors are included in this PR. Most of the substantial stuff is concentrated in 5-6 files and is fairly well split out into separate commits. To weed out refactors from feature work, though, you might find this easier to review commit by commit.

Author: ndhoule

package-lock.json

@@ -1,11 +1,10 @@
-import { promises as fs } from 'fs'
-import { normalize, resolve } from 'path'
+import { promises as fs } from 'node:fs'
+import { normalize } from 'node:path'
+import { fileURLToPath } from 'node:url'
 
-import { execa } from 'execa'
 import { pathExists } from 'path-exists'
 import { isFile } from 'path-type'
 
-import { logArray, logSubHeader } from '../log/logger.js'
 import { logInstallMissingPlugins, logInstallIntegrations } from '../log/messages/install.js'
 
 import { addExactDependencies } from './main.js'
@@ -37,71 +36,38 @@ export const installIntegrationPlugins = async function ({
   pluginsEnv,
   buildDir,
 }) {
-  const integrationsToBuild = integrations.filter(
-    (integration) => typeof integration.dev !== 'undefined' && context === 'dev',
-  )
-  if (integrationsToBuild.length) {
-    logSubHeader(logs, 'Building extensions')
-    logArray(
-      logs,
-      integrationsToBuild.map(({ slug, dev: { path } }) => `${slug} from ${path}`),
-    )
-  }
   const packages = (
     await Promise.all(
       integrations.map((integration) =>
         getIntegrationPackage({ integration, context, testOpts, buildDir, pluginsEnv }),
       ),
     )
   ).filter(Boolean)
-  logInstallIntegrations(
-    logs,
-    integrations.filter((integration) =>
-      integrationsToBuild.every((compiledIntegration) => integration.slug !== compiledIntegration.slug),
-    ),
-  )
+  logInstallIntegrations(logs, integrations)
 
   if (packages.length === 0) {
     return
   }
 
   await createAutoPluginsDir(logs, autoPluginsDir)
-
   await addExactDependencies({ packageRoot: autoPluginsDir, isLocal: mode !== 'buildbot', packages })
 }
 
-const getIntegrationPackage = async function ({
-  integration: { version, dev },
-  context,
-  testOpts = {},
-  buildDir,
-  pluginsEnv,
-}) {
-  if (typeof version !== 'undefined') {
-    return `${version}/packages/buildhooks.tgz`
-  }
-
-  if (typeof dev !== 'undefined' && context === 'dev') {
-    const { path } = dev
-
-    const integrationDir = testOpts.cwd ? resolve(testOpts.cwd, path) : resolve(buildDir, path)
-
-    try {
-      const res = await execa('npm', ['run', 'build'], { cwd: integrationDir, env: pluginsEnv })
-
-      // This is horrible and hacky, but `npm run build` will
-      // return status code 0 even if it fails
-      if (!res.stdout.includes('Build complete!')) {
-        throw new Error(res.stdout)
-      }
-    } catch (e) {
-      throw new Error(`Failed to build integration. Error:\n\n${e.stack}`)
-    }
-
+const getIntegrationPackage = async function ({ integration: { buildPlugin } }) {
+  if (buildPlugin === null) {
     return undefined
   }
 
-  return undefined
+  switch (buildPlugin.packageURL.protocol) {
+    case 'http:':
+    // fallthrough
+    case 'https:':
+      return buildPlugin.packageURL.toString()
+    case 'file:':
+      return fileURLToPath(buildPlugin.packageURL)
+    default:
+      throw new Error(`unsupported build plugin package URL: ${buildPlugin.packageURL.toString()}`)
+  }
 }
 
 // We pin the version without using semver ranges ^ nor ~

packages/build/src/install/missing.js

@@ -1,4 +1,4 @@
-import { join, resolve } from 'path'
+import { join } from 'node:path'
 
 import { addErrorInfo } from '../error/info.js'
 import { installMissingPlugins, installIntegrationPlugins } from '../install/missing.js'
@@ -66,9 +66,7 @@ export const resolvePluginsPath = async function ({
     logs,
   })
 
-  let integrationPluginOptions = []
-
-  integrationPluginOptions = await handleIntegrations({
+  const integrationPluginOptions = await handleIntegrations({
     integrations,
     autoPluginsDir,
     mode,
@@ -175,9 +173,9 @@ const handleIntegrations = async function ({
   testOpts,
   pluginsEnv,
 }) {
-  const toInstall = integrations.filter((integration) => integration.has_build)
+  const integrationsWithBuildPlugins = integrations.filter((integration) => integration.has_build)
   await installIntegrationPlugins({
-    integrations: toInstall,
+    integrations: integrationsWithBuildPlugins,
     autoPluginsDir,
     mode,
     logs,
@@ -186,38 +184,29 @@ const handleIntegrations = async function ({
     buildDir,
     pluginsEnv,
   })
-
-  if (toInstall.length === 0) {
-    return []
-  }
-
   return Promise.all(
-    toInstall.map((integration) =>
-      resolveIntegration({
+    integrationsWithBuildPlugins.map(async (integration) => {
+      // TODO(ndhoule): When developing locally, the user can accidentally set an extension name in
+      // their netlify.toml that points to a build plugin package that has a mismatched name. This
+      // will result in a failed install here. The solution is non-obvious: make sure your
+      // `netlify.toml#integrations[].name` matches the extension that `netlify.toml#integrations[].dev.path`
+      // points at.
+      //
+      // (We could, for example, detect a mismatch by untarring the local build plugin package in
+      // memory and comparing its `package.json#name` to `integration.slug`, and throw a descriptive
+      // error if they don't match.)
+      const packageName = `${integration.slug}-buildhooks`
+      return {
         integration,
-        autoPluginsDir,
-        buildDir,
-        context,
-        testOpts,
-      }),
-    ),
+        isIntegration: true,
+        loadedFrom: integration.buildPlugin.origin === 'local' ? 'local' : undefined,
+        packageName,
+        pluginPath: await resolvePath(packageName, autoPluginsDir),
+      }
+    }),
   )
 }
 
-const resolveIntegration = async function ({ integration, autoPluginsDir, buildDir, context, testOpts }) {
-  if (typeof integration.dev !== 'undefined' && context === 'dev') {
-    const { path } = integration.dev
-    const integrationDir = testOpts.cwd ? resolve(testOpts.cwd, path) : resolve(buildDir, path)
-    const pluginPath = await resolvePath(`${integrationDir}/.ntli/build`, buildDir)
-
-    return { pluginPath, packageName: `${integration.slug}`, isIntegration: true, integration, loadedFrom: 'local' }
-  }
-
-  const pluginPath = await resolvePath(`${integration.slug}-buildhooks`, autoPluginsDir)
-
-  return { pluginPath, packageName: `${integration.slug}-buildhooks`, isIntegration: true, integration }
-}
-
 // Resolve the plugins that just got automatically installed
 const resolveMissingPluginPath = async function ({ pluginOptions, pluginOptions: { packageName }, autoPluginsDir }) {
   if (!isMissingPlugin(pluginOptions)) {