feat: Add db init and db status commands (#7115)

* POC adding database init command / drizzle-kit as an external subcommand

* cleanup

* cleanup

* fix awaiting drizzle deps installing / add status check and getSiteConfiguration

* fix request headers for init endpoint / misc cleanup

* only inquirer.prompt if initialOpts.drizzle is not explicitly passed as true or false

* move urls to constants with env overrides / cleanup logging / remove unused function / add --yes option to use defaults and --overwrite to overwrite files

* remove commented unused call to get site

* remove unnecessary check

* fix: change slug into the neon slug

* Update db init command (#7257)

* feat: add local dev branch option

* add UNPOOLED env to db status command

* add nf-db-user-id to req headers and update endpoint url

* fix name for NETLIFY_DATABASE_URL_UNPOOLED

* update NEON_DATABASE_EXTENSION_SLUG to 'neon' and remove NETLIFY_WEB_UI constant

* remove dev command and dev-branch.ts

* init: remove/replace headers for Nf-UIExt headers / remove dev branch questions

* update getExtension, installExtension to call jigsaw directly instead of calling react ui endpoints

* token -> netlifyToken

* drizzle - fixes/cleanup & remove dev branch config

* remove localDevBranch from Answers type

---------

Co-authored-by: Karin <=>

* update db command help text

* update command descriptions and examples

* update help command snapshot

* support docs gen for sub commands without ":"

* docs gen for db commands

* remove "yes" flag and add "minimal" flag

* improve db init ux - dont throw error on CONFLICT (db already connected)

- if db already connected, we just say its connected and continue to log
the status
- improve status by fetching cli-db-status extension endpoint

* install @netlify/neon package if not found in package.json

* use same package json path

* move neon package installation to end to avoid incorrect overwriting

* add comment to drizzle config boilerplate for context

* fix: initDrizzle - fallback to command.project.baseDirectory when command.project.root not found

* perf: lazy-load `netlify db` commands

* Update src/commands/database/constants.ts

* feat: get user to initialize site first if they have neon installed for dev (#7319)

* feat: add handling of extension requirements except doing nothing but logging

* feat: require users to init a site in dev if they have neon package installed

* chore: better comment

* chore: format

* feat: move back to logical or

* refactor: address review comments

---------

Co-authored-by: Nathan Houle <[email protected]>

* feat: rename --minimal => --assume-no, --drizzle => --boilerplate=<type>

* docs: update docs with new db flags

* fix: update db init example to not use --minimal

* docs: regenerate docs again

---------

Co-authored-by: Karin <=>
Co-authored-by: Nathan Houle <[email protected]>
Co-authored-by: Sarah Etter <[email protected]>
Co-authored-by: Karin Hendrikse <[email protected]>

Author: 4 people

docs/commands/db.md

@@ -0,0 +1,86 @@
+---
+title: Netlify CLI db command
+description: Provision a production ready Postgres database with a single command
+sidebar:
+  label: db
+---
+
+# `db`
+
+
+<!-- AUTO-GENERATED-CONTENT:START (GENERATE_COMMANDS_DOCS) -->
+Provision a production ready Postgres database with a single command
+
+**Usage**
+
+```bash
+netlify db
+```
+
+**Flags**
+
+- `filter` (*string*) - For monorepos, specify the name of the application to run the command in
+- `debug` (*boolean*) - Print debugging information
+- `auth` (*string*) - Netlify auth token - can be used to run this command without logging in
+
+| Subcommand | description  |
+|:--------------------------- |:-----|
+| [`init`](/commands/db#init) | Initialize a new database for the current site  |
+| [`status`](/commands/db#status) | Check the status of the database  |
+
+
+**Examples**
+
+```bash
+netlify db status
+netlify db init
+netlify db init --help
+```
+
+---
+## `init`
+
+Initialize a new database for the current site
+
+**Usage**
+
+```bash
+netlify init
+```
+
+**Flags**
+
+- `assume-no` (*boolean*) - Non-interactive setup. Does not initialize any third-party tools/boilerplate. Ideal for CI environments or AI tools.
+- `boilerplate` (*drizzle*) - Type of boilerplate to add to your project.
+- `filter` (*string*) - For monorepos, specify the name of the application to run the command in
+- `overwrite` (*boolean*) - Overwrites existing files that would be created when setting up boilerplate
+- `debug` (*boolean*) - Print debugging information
+- `auth` (*string*) - Netlify auth token - can be used to run this command without logging in
+- `no-boilerplate` (*boolean*) - Don't add any boilerplate to your project.
+
+**Examples**
+
+```bash
+netlify db init --assume-no
+netlify db init --boilerplate=drizzle --overwrite
+```
+
+---
+## `status`
+
+Check the status of the database
+
+**Usage**
+
+```bash
+netlify status
+```
+
+**Flags**
+
+- `debug` (*boolean*) - Print debugging information
+- `auth` (*string*) - Netlify auth token - can be used to run this command without logging in
+
+---
+
+<!-- AUTO-GENERATED-CONTENT:END -->

docs/index.md

@@ -51,6 +51,16 @@ Generate shell completion script
 | [`completion:install`](/commands/completion#completioninstall) | Generates completion script for your preferred shell  |
 
 
+### [db](/commands/db)
+
+Provision a production ready Postgres database with a single command
+
+| Subcommand | description  |
+|:--------------------------- |:-----|
+| [`init`](/commands/db#init) | Initialize a new database for the current site  |
+| [`status`](/commands/db#status) | Check the status of the database  |
+
+
 ### [deploy](/commands/deploy)
 
 Create a new deploy from the contents of a folder

site/scripts/docs.js

@@ -100,7 +100,11 @@ const commandListSubCommandDisplay = function (commands) {
   let table = '| Subcommand | description  |\n'
   table += '|:--------------------------- |:-----|\n'
   commands.forEach((cmd) => {
-    const [commandBase] = cmd.name.split(':')
+    let commandBase
+    commandBase = cmd.name.split(':')[0]
+    if (cmd.parent) {
+      commandBase = cmd.parent
+    }
     const baseUrl = `/commands/${commandBase}`
     const slug = cmd.name.replace(/:/g, '')
     table += `| [\`${cmd.name}\`](${baseUrl}#${slug}) | ${cmd.description.split('\n')[0]}  |\n`

site/scripts/util/generate-command-data.js

@@ -35,12 +35,13 @@ const parseCommand = function (command) {
     }, {})
 
   return {
+    parent: command.parent?.name() !== "netlify" ? command.parent?.name() : undefined,
     name: command.name(),
     description: command.description(),
-    commands: commands
-
-      .filter((cmd) => cmd.name().startsWith(`${command.name()}:`) && !cmd._hidden)
-      .map((cmd) => parseCommand(cmd)),
+    commands: [
+      ...command.commands.filter(cmd => !cmd._hidden).map(cmd => parseCommand(cmd)),
+      ...commands.filter((cmd) => cmd.name().startsWith(`${command.name()}:`) && !cmd._hidden).map(cmd => parseCommand(cmd))
+    ],
     examples: command.examples.length !== 0 && command.examples,
     args: args.length !== 0 && args,
     flags: Object.keys(flags).length !== 0 && flags,

src/commands/base-command.ts

@@ -14,6 +14,7 @@ import { findUp } from 'find-up'
 import inquirer from 'inquirer'
 import inquirerAutocompletePrompt from 'inquirer-autocomplete-prompt'
 import merge from 'lodash/merge.js'
+import pick from 'lodash/pick.js'
 
 import { getAgent } from '../lib/http-agent.js'
 import {
@@ -785,3 +786,6 @@ export default class BaseCommand extends Command {
     return this.netlify.siteInfo.feature_flags?.[flagName] || null
   }
 }
+
+export const getBaseOptionValues = (options: OptionValues): BaseOptionValues =>
+  pick(options, ['auth', 'cwd', 'debug', 'filter', 'httpProxy', 'silent'])

src/commands/database/constants.ts

@@ -0,0 +1,3 @@
+export const NEON_DATABASE_EXTENSION_SLUG = process.env.NEON_DATABASE_EXTENSION_SLUG ?? 'neon'
+export const JIGSAW_URL = process.env.JIGSAW_URL ?? 'https://jigsaw.services-prod.nsvcs.net'
+export const NETLIFY_NEON_PACKAGE_NAME = '@netlify/neon'

src/commands/database/database.ts

@@ -0,0 +1,83 @@
+import { Option } from 'commander'
+import inquirer from 'inquirer'
+import BaseCommand from '../base-command.js'
+import type { DatabaseBoilerplateType, DatabaseInitOptions } from './init.js'
+
+export type Extension = {
+  id: string
+  name: string
+  slug: string
+  hostSiteUrl: string
+  installedOnTeam: boolean
+}
+
+export type SiteInfo = {
+  id: string
+  name: string
+  account_id: string
+  admin_url: string
+  url: string
+  ssl_url: string
+}
+
+const supportedBoilerplates = new Set<DatabaseBoilerplateType>(['drizzle'])
+
+export const createDatabaseCommand = (program: BaseCommand) => {
+  const dbCommand = program
+    .command('db')
+    .alias('database')
+    .description(`Provision a production ready Postgres database with a single command`)
+    .addExamples(['netlify db status', 'netlify db init', 'netlify db init --help'])
+
+  dbCommand
+    .command('init')
+    .description(`Initialize a new database for the current site`)
+    .option(
+      '--assume-no',
+      'Non-interactive setup. Does not initialize any third-party tools/boilerplate. Ideal for CI environments or AI tools.',
+      false,
+    )
+    .addOption(
+      new Option('--boilerplate <tool>', 'Type of boilerplate to add to your project.').choices(
+        Array.from(supportedBoilerplates).sort(),
+      ),
+    )
+    .option('--no-boilerplate', "Don't add any boilerplate to your project.")
+    .option('-o, --overwrite', 'Overwrites existing files that would be created when setting up boilerplate')
+    .action(async (_options: Record<string, unknown>, command: BaseCommand) => {
+      const { init } = await import('./init.js')
+
+      // Only prompt for drizzle if the user did not specify a boilerplate option, and if we're in
+      // interactive mode
+      if (_options.boilerplate === undefined && !_options.assumeNo) {
+        const answers = await inquirer.prompt<{ useDrizzle: boolean }>([
+          {
+            type: 'confirm',
+            name: 'useDrizzle',
+            message: 'Set up Drizzle boilerplate?',
+          },
+        ])
+        if (answers.useDrizzle) {
+          command.setOptionValue('boilerplate', 'drizzle')
+        }
+      }
+
+      const options = _options as DatabaseInitOptions
+      if (options.assumeNo) {
+        options.boilerplate = false
+        options.overwrite = false
+      }
+
+      await init(options, command)
+    })
+    .addExamples([`netlify db init --assume-no`, `netlify db init --boilerplate=drizzle --overwrite`])
+
+  dbCommand
+    .command('status')
+    .description(`Check the status of the database`)
+    .action(async (options, command) => {
+      const { status } = await import('./status.js')
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      await status(options, command)
+    })
+}

src/commands/database/drizzle.ts

@@ -0,0 +1,112 @@
+import { carefullyWriteFile, getPackageJSON, spawnAsync } from './utils.js'
+import BaseCommand from '../base-command.js'
+import path from 'path'
+import fs from 'fs/promises'
+import inquirer from 'inquirer'
+import { NETLIFY_NEON_PACKAGE_NAME } from './constants.js'
+
+export const initDrizzle = async (command: BaseCommand) => {
+  const workingDirectory = command.project.root ?? command.project.baseDirectory
+  if (!workingDirectory) {
+    throw new Error('Failed to initialize Drizzle. Project root or base directory not found.')
+  }
+  const opts = command.opts<{
+    overwrite?: true | undefined
+  }>()
+
+  const drizzleConfigFilePath = path.resolve(workingDirectory, 'drizzle.config.ts')
+  const schemaFilePath = path.resolve(workingDirectory, 'db/schema.ts')
+  const dbIndexFilePath = path.resolve(workingDirectory, 'db/index.ts')
+  if (opts.overwrite) {
+    await fs.writeFile(drizzleConfigFilePath, drizzleConfig)
+    await fs.mkdir(path.resolve(workingDirectory, 'db'), { recursive: true })
+    await fs.writeFile(schemaFilePath, drizzleSchema)
+    await fs.writeFile(dbIndexFilePath, dbIndex)
+  } else {
+    await carefullyWriteFile(drizzleConfigFilePath, drizzleConfig, workingDirectory)
+    await fs.mkdir(path.resolve(workingDirectory, 'db'), { recursive: true })
+    await carefullyWriteFile(schemaFilePath, drizzleSchema, workingDirectory)
+    await carefullyWriteFile(dbIndexFilePath, dbIndex, workingDirectory)
+  }
+
+  const packageJsonPath = path.resolve(command.workingDir, 'package.json')
+  const packageJson = getPackageJSON(command.workingDir)
+
+  packageJson.scripts = {
+    ...(packageJson.scripts ?? {}),
+    ...packageJsonScripts,
+  }
+  if (opts.overwrite) {
+    await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2))
+  }
+
+  type Answers = {
+    updatePackageJson: boolean
+  }
+
+  if (!opts.overwrite) {
+    const answers = await inquirer.prompt<Answers>([
+      {
+        type: 'confirm',
+        name: 'updatePackageJson',
+        message: `Add drizzle db commands to package.json?`,
+      },
+    ])
+    if (answers.updatePackageJson) {
+      await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2))
+    }
+  }
+
+  if (!Object.keys(packageJson.devDependencies ?? {}).includes('drizzle-kit')) {
+    await spawnAsync(command.project.packageManager?.installCommand ?? 'npm install', ['drizzle-kit@latest', '-D'], {
+      stdio: 'inherit',
+      shell: true,
+    })
+  }
+
+  if (!Object.keys(packageJson.dependencies ?? {}).includes('drizzle-orm')) {
+    await spawnAsync(command.project.packageManager?.installCommand ?? 'npm install', ['drizzle-orm@latest'], {
+      stdio: 'inherit',
+      shell: true,
+    })
+  }
+}
+
+const drizzleConfig = `import { defineConfig } from 'drizzle-kit';
+
+export default defineConfig({
+    dialect: 'postgresql',
+    dbCredentials: {
+        url: process.env.NETLIFY_DATABASE_URL!
+    },
+    schema: './db/schema.ts',
+    /**
+     * Never edit the migrations directly, only use drizzle.
+     * There are scripts in the package.json "db:generate" and "db:migrate" to handle this.
+     */
+    out: './migrations'
+});`
+
+const drizzleSchema = `import { integer, pgTable, varchar, text } from 'drizzle-orm/pg-core';
+
+export const posts = pgTable('posts', {
+    id: integer().primaryKey().generatedAlwaysAsIdentity(),
+    title: varchar({ length: 255 }).notNull(),
+    content: text().notNull().default('')
+});`
+
+const dbIndex = `import { neon } from '${NETLIFY_NEON_PACKAGE_NAME}';
+import { drizzle } from 'drizzle-orm/neon-http';
+
+import * as schema from './schema';
+
+export const db = drizzle({
+    schema,
+    client: neon()
+});`
+
+const packageJsonScripts = {
+  'db:generate': 'drizzle-kit generate',
+  'db:migrate': 'netlify dev:exec drizzle-kit migrate',
+  'db:studio': 'netlify dev:exec drizzle-kit studio',
+}

src/commands/database/index.ts

@@ -0,0 +1 @@
+export { createDatabaseCommand } from './database.js'