feat: support granular context file setting (#7284)

* feat: automatically detects IDE - windsurf or cursor - for AI context files.

* Don't ask permission to write ai-context files if IDE was detected.

Added `--skip-detection` flag to skip the automatic IDE detection and fallback to choice list.

* PR review fixes

* remove unused options field

* remove unused recipeName field

* remove unused options field

* formatting

* feat: support granular context file setting

* handle docs site not being available

* fix: nullish coalescing

* chore: moving download/write function to a more testable location

* adding testing for ai-context downloading

* feat: updating granular context to have test coverage

* chore: ensure mocks are all reset after tests

* fix: formatting and parse tests

* fix: fix the test for windows paths

* fix: fix the test for windows paths

---------

Co-authored-by: Simon Hanukaev <[email protected]>
Co-authored-by: Sean Roberts <[email protected]>

Author: 3 people

src/recipes/ai-context/context.ts

@@ -1,22 +1,94 @@
 import { promises as fs } from 'node:fs'
-import { dirname } from 'node:path'
+import { dirname, resolve } from 'node:path'
+import semver from 'semver'
+import { chalk, logAndThrowError, log, version } from '../../utils/command-helpers.js'
+import type { RunRecipeOptions } from '../../commands/recipes/recipes.js'
 
 const ATTRIBUTES_REGEX = /(\S*)="([^\s"]*)"/gim
-const BASE_URL = 'https://docs.netlify.com/ai-context'
-export const FILE_NAME = 'netlify-development.mdc'
+// AI_CONTEXT_BASE_URL is used to help with local testing at non-production
+// versions of the context apis.
+const BASE_URL = new URL(process.env.AI_CONTEXT_BASE_URL ?? 'https://docs.netlify.com/ai-context').toString()
+export const NTL_DEV_MCP_FILE_NAME = 'netlify-development.mdc'
 const MINIMUM_CLI_VERSION_HEADER = 'x-cli-min-ver'
 export const NETLIFY_PROVIDER = 'netlify'
 const PROVIDER_CONTEXT_REGEX = /<providercontext ([^>]*)>(.*)<\/providercontext>/ims
 const PROVIDER_CONTEXT_OVERRIDES_REGEX = /<providercontextoverrides([^>]*)>(.*)<\/providercontextoverrides>/ims
 const PROVIDER_CONTEXT_OVERRIDES_TAG = 'ProviderContextOverrides'
 
-export const downloadFile = async (cliVersion: string) => {
+export interface ContextConfig {
+  scope: string
+  glob?: string
+  shared?: string[]
+  endpoint?: string
+}
+
+export interface ContextFile {
+  key: string
+  config: ContextConfig
+  content: string
+}
+
+export interface ConsumerConfig {
+  key: string
+  presentedName: string
+  consumerProcessCmd?: string
+  path: string
+  ext: string
+  truncationLimit?: number
+  contextScopes: Record<string, ContextConfig>
+  hideFromCLI?: boolean
+  consumerTrigger?: string
+}
+
+let contextConsumers: ConsumerConfig[] = []
+export const getContextConsumers = async (cliVersion: string) => {
+  if (contextConsumers.length > 0) {
+    return contextConsumers
+  }
   try {
-    const res = await fetch(`${BASE_URL}/${FILE_NAME}`, {
+    const res = await fetch(`${BASE_URL}/context-consumers`, {
       headers: {
         'user-agent': `NetlifyCLI ${cliVersion}`,
       },
     })
+
+    if (!res.ok) {
+      return []
+    }
+
+    const data = (await res.json()) as { consumers: ConsumerConfig[] } | undefined
+    contextConsumers = data?.consumers ?? []
+  } catch {}
+
+  return contextConsumers
+}
+
+export const downloadFile = async (cliVersion: string, contextConfig: ContextConfig, consumer: ConsumerConfig) => {
+  try {
+    if (!contextConfig.endpoint) {
+      return null
+    }
+
+    const url = new URL(contextConfig.endpoint, BASE_URL)
+    url.searchParams.set('consumer', consumer.key)
+
+    if (process.env.AI_CONTEXT_BASE_URL) {
+      const overridingUrl = new URL(process.env.AI_CONTEXT_BASE_URL)
+      url.host = overridingUrl.host
+      url.port = overridingUrl.port
+      url.protocol = overridingUrl.protocol
+    }
+
+    const res = await fetch(url, {
+      headers: {
+        'user-agent': `NetlifyCLI ${cliVersion}`,
+      },
+    })
+
+    if (!res.ok) {
+      return null
+    }
+
     const contents = await res.text()
     const minimumCLIVersion = res.headers.get(MINIMUM_CLI_VERSION_HEADER) ?? undefined
 
@@ -99,10 +171,12 @@ export const applyOverrides = (template: string, overrides?: string) => {
     return template
   }
 
-  return template.replace(
-    PROVIDER_CONTEXT_OVERRIDES_REGEX,
-    `<${PROVIDER_CONTEXT_OVERRIDES_TAG}>${overrides}</${PROVIDER_CONTEXT_OVERRIDES_TAG}>`,
-  )
+  return template
+    .replace(
+      PROVIDER_CONTEXT_OVERRIDES_REGEX,
+      `<${PROVIDER_CONTEXT_OVERRIDES_TAG}>${overrides}</${PROVIDER_CONTEXT_OVERRIDES_TAG}>`,
+    )
+    .trim()
 }
 
 /**
@@ -137,3 +211,84 @@ export const writeFile = async (path: string, contents: string) => {
   await fs.mkdir(directory, { recursive: true })
   await fs.writeFile(path, contents)
 }
+
+export const deleteFile = async (path: string) => {
+  try {
+    // delete file from file system - not just unlinking it
+    await fs.rm(path)
+  } catch {
+    // ignore
+  }
+}
+
+export const downloadAndWriteContextFiles = async (consumer: ConsumerConfig, { command }: RunRecipeOptions) => {
+  await Promise.allSettled(
+    Object.keys(consumer.contextScopes).map(async (contextKey) => {
+      const contextConfig = consumer.contextScopes[contextKey]
+
+      const { contents: downloadedFile, minimumCLIVersion } =
+        (await downloadFile(version, contextConfig, consumer).catch(() => null)) ?? {}
+
+      if (!downloadedFile) {
+        return logAndThrowError(
+          `An error occurred when pulling the latest context file for scope ${contextConfig.scope}. Please try again.`,
+        )
+      }
+      if (minimumCLIVersion && semver.lt(version, minimumCLIVersion)) {
+        return logAndThrowError(
+          `This command requires version ${minimumCLIVersion} or above of the Netlify CLI. Refer to ${chalk.underline(
+            'https://ntl.fyi/update-cli',
+          )} for information on how to update.`,
+        )
+      }
+
+      const absoluteFilePath = resolve(
+        command?.workingDir ?? '',
+        consumer.path,
+        `netlify-${contextKey}.${consumer.ext || 'mdc'}`,
+      )
+
+      const existing = await getExistingContext(absoluteFilePath)
+      const remote = parseContextFile(downloadedFile)
+
+      let { contents } = remote
+
+      // Does a file already exist at this path?
+      if (existing) {
+        // If it's a file we've created, let's check the version and bail if we're
+        // already on the latest, otherwise rewrite it with the latest version.
+        if (existing.provider?.toLowerCase() === NETLIFY_PROVIDER) {
+          if (remote.version === existing.version) {
+            log(
+              `You're all up to date! ${chalk.underline(
+                absoluteFilePath,
+              )} contains the latest version of the context files.`,
+            )
+            return
+          }
+
+          // We must preserve any overrides found in the existing file.
+          contents = applyOverrides(remote.contents, existing.overrides?.innerContents)
+        } else {
+          // Whatever exists in the file goes in the overrides block.
+          contents = applyOverrides(remote.contents, existing.contents)
+        }
+      }
+
+      // we don't want to cut off content, but if we _have_ to
+      // then we need to do so before writing or the user's
+      // context gets in a bad state. Note, this can result in
+      // a file that's not parsable next time. This will be
+      // fine because the file will simply be replaced. Not ideal
+      // but solves the issue of a truncated file in a bad state
+      // being updated.
+      if (consumer.truncationLimit && contents.length > consumer.truncationLimit) {
+        contents = contents.slice(0, consumer.truncationLimit)
+      }
+
+      await writeFile(absoluteFilePath, contents)
+
+      log(`${existing ? 'Updated' : 'Created'} context files at ${chalk.underline(absoluteFilePath)}`)
+    }),
+  )
+}