Autocomplete: Split debounce into two chunks, race the second part with the context retrieving (#3149)

This PR splits the debounce time into two logical chunk. The first 25ms
(regardless of how long the debounce time is defined), the behavior will
be like it is now: We batch events and don't do any concurrent work.

However after 25ms (which, remember, was the debounce time we had until
recently), we start to fetch the context in parallel. The idea is that
we have shave off up to 50ms from the context retrieval time.

Here's a visualization of a trace:

<img width="1724" alt="Screenshot 2024-02-13 at 11 37 12"
src="https://github.com/sourcegraph/cody/assets/458591/d8366a19-d0dd-4d20-a2bc-3b301d5e6211">

As you can see in this specific example, context retrieval could almost
be free.

One caveat here is that this _will_ increase CPU pressure (since we
start fetching context a bit earlier). That's why I think 25ms is a good
value for this since we have used it in the past with the same retrieval
algorithm quite successfully.

## Test plan

- Ensure AC still works
- Check out a trace

<img width="1718" alt="Screenshot 2024-02-13 at 11 47 14"
src="https://github.com/sourcegraph/cody/assets/458591/37acfa14-f4a2-421b-813c-c5dd2b25b174">

Author: philipp-spiess

vscode/CHANGELOG.md

@@ -14,6 +14,7 @@ This is a log of all notable changes to Cody for VS Code. [Unreleased] changes a
 
 - Autocomplete: Removes the latency for cached completions. [https://github.com/sourcegraph/cody/pull/3138](https://github.com/sourcegraph/cody/pull/3138)
 - Autocomplete: Enable the recent jaccard similarity improvements by default. [pull/3135](https://github.com/sourcegraph/cody/pull/3135)
+- Autocomplete: Start retrieval phase earlier to improve latency. [pull/3149](https://github.com/sourcegraph/cody/pull/3149)
 
 ## [1.4.3]
 

vscode/src/completions/get-inline-completions-tests/helpers.ts

@@ -32,10 +32,11 @@ import {
 } from '../providers/anthropic'
 import type { ProviderOptions } from '../providers/provider'
 import { RequestManager } from '../request-manager'
-import { documentAndPosition, sleep } from '../test-helpers'
+import { documentAndPosition } from '../test-helpers'
 import { pressEnterAndGetIndentString } from '../providers/hot-streak'
 import { completionProviderConfig } from '../completion-provider-config'
 import { emptyMockFeatureFlagProvider } from '../../testutils/mocks'
+import { sleep } from '../utils'
 
 // The dedent package seems to replace `\t` with `\\t` so in order to insert a tab character, we
 // have to use interpolation. We abbreviate this to `T` because ${T} is exactly 4 characters,

vscode/src/completions/get-inline-completions.ts

@@ -28,6 +28,7 @@ import type { InlineCompletionItemWithAnalytics } from './text-processing/proces
 import type { ProvideInlineCompletionsItemTraceData } from './tracer'
 import { isValidTestFile } from '../commands/utils/test-commands'
 import { completionProviderConfig } from './completion-provider-config'
+import { sleep } from './utils'
 
 export interface InlineCompletionsParams {
     // Context
@@ -306,6 +307,21 @@ async function doGetInlineCompletions(
         }
     }
 
+    const debounceTime =
+        triggerKind !== TriggerKind.Automatic
+            ? 0
+            : ((multiline ? debounceInterval?.multiLine : debounceInterval?.singleLine) ?? 0) +
+              (artificialDelay ?? 0)
+
+    // We split the desired debounceTime into two chunks. One that is at most 25ms where every
+    // further execution is halted...
+    const waitInterval = Math.min(debounceTime, 25)
+    // ...and one for the remaining time where we can already start retrieving context in parallel.
+    const remainingInterval = debounceTime - waitInterval
+    if (waitInterval > 0) {
+        await wrapInActiveSpan('autocomplete.debounce.wait', () => sleep(waitInterval))
+    }
+
     // Debounce to avoid firing off too many network requests as the user is still typing.
     await wrapInActiveSpan('autocomplete.debounce', async () => {
         const interval =
@@ -324,19 +340,26 @@ async function doGetInlineCompletions(
     setIsLoading?.(true)
     CompletionLogger.start(logId)
 
-    // Fetch context
-    const contextResult = await wrapInActiveSpan('autocomplete.retrieve', async () => {
-        return contextMixer.getContext({
-            document,
-            position,
-            docContext,
-            abortSignal,
-            maxChars: providerConfig.contextSizeHints.totalChars,
-        })
-    })
+    // Fetch context and apply remaining debounce time
+    const [contextResult] = await Promise.all([
+        wrapInActiveSpan('autocomplete.retrieve', () =>
+            contextMixer.getContext({
+                document,
+                position,
+                docContext,
+                abortSignal,
+                maxChars: providerConfig.contextSizeHints.totalChars,
+            })
+        ),
+        remainingInterval > 0
+            ? wrapInActiveSpan('autocomplete.debounce.remaining', () => sleep(remainingInterval))
+            : null,
+    ])
+
     if (abortSignal?.aborted) {
         return null
     }
+
     tracer?.({ context: contextResult })
 
     const completionProvider = getCompletionProvider({

vscode/src/completions/test-helpers.ts

@@ -60,7 +60,3 @@ export function documentAndPosition(
 export function nextTick(): Promise<void> {
     return new Promise(resolve => setTimeout(resolve, 0))
 }
-
-export function sleep(ms: number): Promise<void> {
-    return new Promise(resolve => setTimeout(resolve, ms))
-}

vscode/src/completions/utils.ts

@@ -131,3 +131,7 @@ function createTimeout(timeoutMs: number): Promise<never> {
         setTimeout(() => reject(new TimeoutError('The request timed out')), timeoutMs)
     )
 }
+
+export function sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms))
+}