add ghes support to Copilot Search (#55487)

Author: Ebonsignori

src/events/components/experiments/experiments.ts

@@ -24,11 +24,6 @@ export const EXPERIMENTS = {
     percentOfUsersToGetExperiment: 0, // 10% of users will get the experiment
     includeVariationInContext: true, // All events will include the `experiment_variation` of the `ai_search_experiment`
     limitToLanguages: ['en'], // Only users with the `en` language will be included in the experiment
-    limitToVersions: [
-      'free-pro-team@latest',
-      'enterprise-cloud@latest',
-      'enterprise-server@latest',
-    ], // Only enable for versions
     alwaysShowForStaff: true, // When set to true, staff will always see the experiment (determined by the `staffonly` cookie)
     turnOnWithURLParam: 'ai_search', /// When the query param `?feature=ai_search` is set, the experiment will be enabled
   },

src/search/components/helpers/execute-search-actions.ts

@@ -45,18 +45,10 @@ export function executeGeneralSearch(
   router.push(asPath, undefined, { shallow: false })
 }
 
-export async function executeAISearch(
-  router: NextRouter,
-  version: string,
-  query: string,
-  debug = false,
-) {
-  let language = router.locale || 'en'
-
+export async function executeAISearch(version: string, query: string, debug = false) {
   const body = {
     query,
     version,
-    language,
     ...(debug && { debug: '1' }),
   }
 

src/search/components/input/AskAIResults.tsx

@@ -167,7 +167,7 @@ export function AskAIResults({
       let conversationIdBuffer = ''
 
       try {
-        const response = await executeAISearch(router, version, query, debug)
+        const response = await executeAISearch(version, query, debug)
         if (!response.ok) {
           // If there is JSON and the `upstreamStatus` key, the error is from the upstream sever (CSE)
           let responseJson

src/search/lib/ai-search-proxy.ts

@@ -1,11 +1,12 @@
-import { Request, Response } from 'express'
+import { Response } from 'express'
 import statsd from '@/observability/lib/statsd'
 import got from 'got'
 import { getHmacWithEpoch } from '@/search/lib/helpers/get-cse-copilot-auth'
 import { getCSECopilotSource } from '@/search/lib/helpers/cse-copilot-docs-versions'
+import type { ExtendedRequest } from '@/types'
 
-export const aiSearchProxy = async (req: Request, res: Response) => {
-  const { query, version, language } = req.body
+export const aiSearchProxy = async (req: ExtendedRequest, res: Response) => {
+  const { query, version } = req.body
 
   const errors = []
 
@@ -15,18 +16,12 @@ export const aiSearchProxy = async (req: Request, res: Response) => {
   } else if (typeof query !== 'string') {
     errors.push({ message: `Invalid 'query' in request body. Must be a string` })
   }
-  if (!version) {
-    errors.push({ message: `Missing required key 'version' in request body` })
-  }
-  if (!language) {
-    errors.push({ message: `Missing required key 'language' in request body` })
-  }
 
   let docsSource = ''
   try {
-    docsSource = getCSECopilotSource(version, language)
+    docsSource = getCSECopilotSource(version)
   } catch (error: any) {
-    errors.push({ message: error?.message || 'Invalid version or language' })
+    errors.push({ message: error?.message || 'Invalid version' })
   }
 
   if (errors.length) {
@@ -36,7 +31,7 @@ export const aiSearchProxy = async (req: Request, res: Response) => {
 
   const diagnosticTags = [
     `version:${version}`.slice(0, 200),
-    `language:${language}`.slice(0, 200),
+    `language:${req.language}`.slice(0, 200),
     `queryLength:${query.length}`.slice(0, 200),
   ]
   statsd.increment('ai-search.call', 1, diagnosticTags)
@@ -52,7 +47,8 @@ export const aiSearchProxy = async (req: Request, res: Response) => {
   }
 
   try {
-    const stream = got.stream.post(`${process.env.CSE_COPILOT_ENDPOINT}/answers`, {
+    // TODO: We temporarily add ?ai_search=1 to use a new pattern in cgs-copilot production
+    const stream = got.stream.post(`${process.env.CSE_COPILOT_ENDPOINT}/answers?ai_search=1`, {
       json: body,
       headers: {
         Authorization: getHmacWithEpoch(),

src/search/lib/elasticsearch-indexes.ts

@@ -74,7 +74,15 @@ export function getElasticSearchIndex(
   }
 
   // e.g. free-pro-team becomes fpt for the index name
-  const indexVersion = versionToIndexVersionMap[version]
+  let indexVersion = versionToIndexVersionMap[version]
+
+  // TODO: For AI Search, we initially only supported the latest GHES version
+  // Supporting more versions would involve adding more indexes and generating the data to fill them
+  // As a work around, we will just use the latest version for all GHES suggestions / autocomplete
+  // This is a temporary fix until we can support more versions
+  if (type === 'aiSearchAutocomplete' && indexVersion.startsWith('ghes')) {
+    indexVersion = versionToIndexVersionMap['enterprise-server']
+  }
 
   // In the index-test-fixtures.sh script, we use the tests_ prefix index for testing
   const testPrefix = process.env.NODE_ENV === 'test' ? 'tests_' : ''

src/search/lib/helpers/cse-copilot-docs-versions.ts

@@ -1,5 +1,4 @@
 // Versions used by cse-copilot
-import { allVersions } from '@/versions/lib/all-versions'
 import { versionToIndexVersionMap } from '../elasticsearch-versions'
 const CSE_COPILOT_DOCS_VERSIONS = ['dotcom', 'ghec', 'ghes']
 
@@ -9,70 +8,22 @@ export function supportedCSECopilotLanguages() {
   return DOCS_LANGUAGES
 }
 
-export function getCSECopilotSource(
-  version: (typeof CSE_COPILOT_DOCS_VERSIONS)[number],
-  language: (typeof DOCS_LANGUAGES)[number],
-) {
-  const mappedVersion = versionToIndexVersionMap[version]
-  const { cseCopilotDocsVersion, ghesButNotLatest } = getVersionInfo(mappedVersion)
+export function getCSECopilotSource(version: (typeof CSE_COPILOT_DOCS_VERSIONS)[number]) {
+  if (!version) {
+    throw new Error(`Missing required key 'version' in request body`)
+  }
 
-  if (ghesButNotLatest) {
-    throw new Error(
-      `Only the latest version of GHES is supported for cse-copilot queries. Please use 'ghes@latest'`,
-    )
+  let mappedVersion = versionToIndexVersionMap[version]
+  // CSE-Copilot uses 'dotcom' as the version name for free-pro-team
+  if (mappedVersion === 'fpt') {
+    mappedVersion = 'dotcom'
   }
 
-  if (!CSE_COPILOT_DOCS_VERSIONS.includes(cseCopilotDocsVersion)) {
+  if (!CSE_COPILOT_DOCS_VERSIONS.includes(mappedVersion) && !mappedVersion?.startsWith('ghes-')) {
     throw new Error(
       `Invalid 'version' in request body: '${version}'. Must be one of: ${CSE_COPILOT_DOCS_VERSIONS.join(', ')}`,
     )
   }
-  if (!DOCS_LANGUAGES.includes(language)) {
-    throw new Error(
-      `Invalid 'language' in request body '${language}'. Must be one of: ${DOCS_LANGUAGES.join(', ')}`,
-    )
-  }
-  // cse-copilot uses version names in the form `docs_<shortName>_<language>`, e.g. `docs_ghes_en`
-  return `docs_${cseCopilotDocsVersion}_${language}`
-}
-
-function getVersionInfo(Version: string): {
-  cseCopilotDocsVersion: string
-  ghesButNotLatest: boolean
-} {
-  const versionObject = Object.values(allVersions).find(
-    (info) =>
-      info.shortName === Version ||
-      info.plan === Version ||
-      info.miscVersionName === Version ||
-      info.currentRelease === Version,
-  )
-
-  let cseCopilotDocsVersion = versionObject?.shortName || ''
-  let ghesButNotLatest = false
-  if (!versionObject || !cseCopilotDocsVersion) {
-    return {
-      cseCopilotDocsVersion,
-      ghesButNotLatest,
-    }
-  }
-
-  // CSE-Copilot uses 'dotcom' as the version name for free-pro-team
-  if (cseCopilotDocsVersion === 'fpt') {
-    cseCopilotDocsVersion = 'dotcom'
-  }
-
-  // If ghes, we only support the latest version for cse-copilot queries
-  // Since that's the only version cse-copilot scrapes from our docs
-  if (
-    versionObject.shortName === 'ghes' &&
-    versionObject.currentRelease !== versionObject.latestRelease
-  ) {
-    ghesButNotLatest = true
-  }
-
-  return {
-    cseCopilotDocsVersion,
-    ghesButNotLatest,
-  }
+  // cse-copilot uses version names in the form `docs_<version-shortName>`, e.g. `docs_ghes-3.16`
+  return `docs_${mappedVersion}`
 }

src/search/lib/search-request-params/search-params-objects.ts

@@ -6,7 +6,6 @@
 import languages from '@/languages/lib/languages'
 import { allIndexVersionKeys, versionToIndexVersionMap } from '@/search/lib/elasticsearch-versions'
 import { SearchTypes } from '@/search/types'
-import { latest } from '@/versions/lib/enterprise-server-releases'
 
 import type { SearchRequestQueryParams } from '@/search/lib/search-request-params/types'
 
@@ -120,20 +119,14 @@ const SHARED_AUTOCOMPLETE_PARAMS_OBJ: SearchRequestQueryParams[] = [
     cast: (size: string) => parseInt(size, 10),
     validate: (size: number) => size >= 0 && size <= MAX_AUTOCOMPLETE_SIZE,
   },
-  // We only want to enable for latest versions of fpt, ghec, and ghes
   {
     key: 'version',
     default_: 'free-pro-team',
     validate: (version: string) => {
-      const mappedVersion = versionToIndexVersionMap[version]
-      if (
-        mappedVersion === 'fpt' ||
-        mappedVersion === 'ghec' ||
-        mappedVersion === `ghes-${latest}`
-      ) {
-        return true
+      if (!versionToIndexVersionMap[version]) {
+        throw new ValidationError(`'${version}' not in ${allIndexVersionKeys.join(', ')}`)
       }
-      return false
+      return true
     },
   },
 ]

src/search/tests/api-ai-search.ts

@@ -94,24 +94,8 @@ describe('AI Search Routes', () => {
     ])
   })
 
-  test('should handle validation errors: language missing', async () => {
-    let body = { query: 'example query', version: 'dotcom' }
-    const response = await post('/api/ai-search/v1', {
-      body: JSON.stringify(body),
-      headers: { 'Content-Type': 'application/json' },
-    })
-
-    const responseBody = JSON.parse(response.body)
-
-    expect(response.ok).toBe(false)
-    expect(responseBody['errors']).toEqual([
-      { message: `Missing required key 'language' in request body` },
-      { message: `Invalid 'language' in request body 'undefined'. Must be one of: en` },
-    ])
-  })
-
   test('should handle validation errors: version missing', async () => {
-    let body = { query: 'example query', language: 'en' }
+    let body = { query: 'example query' }
     const response = await post('/api/ai-search/v1', {
       body: JSON.stringify(body),
       headers: { 'Content-Type': 'application/json' },
@@ -122,13 +106,10 @@ describe('AI Search Routes', () => {
     expect(response.ok).toBe(false)
     expect(responseBody['errors']).toEqual([
       { message: `Missing required key 'version' in request body` },
-      {
-        message: `Invalid 'version' in request body: 'undefined'. Must be one of: dotcom, ghec, ghes`,
-      },
     ])
   })
 
-  test('should handle multiple validation errors: query missing, invalid language and version', async () => {
+  test('should handle multiple validation errors: query missing and version', async () => {
     let body = { language: 'fr', version: 'fpt' }
     const response = await post('/api/ai-search/v1', {
       body: JSON.stringify(body),
@@ -140,9 +121,6 @@ describe('AI Search Routes', () => {
     expect(response.ok).toBe(false)
     expect(responseBody['errors']).toEqual([
       { message: `Missing required key 'query' in request body` },
-      {
-        message: `Invalid 'language' in request body 'fr'. Must be one of: en`,
-      },
     ])
   })
 })