fix(codewhisperer): update code coverage tracker to keep tracker of removed characters (#2845)

leigaol · web-flow · commit f4aa2a4c9268 · 2022-08-29T10:09:09.000-07:00
## Problem Current code percentage written metric in CodeCoverageTracker does not count tokens removed by user, which makes this metric inaccurate. ## Solution Similar to what was implemented in AWS Toolkit for JetBrains by @Will-ShaoHua , we make best effort to: * Estimate total user token by counting user inputs and deletions * Estimate total accepted tokens from CodeWhisperer suggestion by counting unmodified suggestions Total user token is updated when there is a TextDocumentChange Event, * If the event is not from human input (i.e. from plugins, formatters), this event is ignored. * If the event is when user copy large chunk of code, this event is ignored. We only count the user manual keyboard input, intelliSense code acceptance and small chunk code copy as inserted total user token. For deletion events, we take the range from them and reduce the total tokens until total tokens reaches 0. Total accepted tokens is updated when user accept a code whisperer suggestion. At that time, the formatted suggestion is counted as accepted tokens. At the time of emitting telemetry data, a edit distance comparision will be done over the remaining code in accepted suggestion range and the accepted suggesetion code to estimate how many characters user has deleted in the accepted suggestion. This edit distance update will also happen when user changes active editor ( go to another file in VS Code).
diff --git a/src/codewhisperer/activation.ts b/src/codewhisperer/activation.ts
@@ -44,6 +44,7 @@ import { SecurityPanelViewProvider } from './views/securityPanelViewProvider'
 import { disposeSecurityDiagnostic } from './service/diagnosticsProvider'
 import { RecommendationHandler } from './service/recommendationHandler'
 import { Commands } from '../shared/vscode/commands2'
+import { CodeWhispererCodeCoverageTracker } from './tracker/codewhispererCodeCoverageTracker'
 
 const performance = globalThis.performance ?? require('perf_hooks').performance
 
@@ -282,6 +283,10 @@ export async function activate(context: ExtContext): Promise<void> {
                     }
                 }
 
+                CodeWhispererCodeCoverageTracker.getTracker(
+                    e.document.languageId,
+                    context.extensionContext.globalState
+                )?.countTotalTokens(e)
                 /**
                  * Handle this keystroke event only when
                  * 1. It is in current non plaintext active editor
@@ -325,6 +330,12 @@ export async function activate(context: ExtContext): Promise<void> {
             }),
             vscode.window.onDidChangeActiveTextEditor(async e => {
                 await InlineCompletion.instance.rejectRecommendation(vscode.window.activeTextEditor)
+                if (vscode.window.activeTextEditor) {
+                    CodeWhispererCodeCoverageTracker.getTracker(
+                        vscode.window.activeTextEditor.document.languageId,
+                        context.extensionContext.globalState
+                    )?.updateAcceptedTokensCount(vscode.window.activeTextEditor)
+                }
             }),
             vscode.window.onDidChangeTextEditorSelection(async e => {
                 if (e.kind === TextEditorSelectionChangeKind.Mouse && vscode.window.activeTextEditor) {
@@ -403,6 +414,11 @@ export async function activate(context: ExtContext): Promise<void> {
                     }
                 }
 
+                CodeWhispererCodeCoverageTracker.getTracker(
+                    e.document.languageId,
+                    context.extensionContext.globalState
+                )?.countTotalTokens(e)
+
                 if (
                     e.document === vscode.window.activeTextEditor?.document &&
                     runtimeLanguageContext.convertLanguage(e.document.languageId) !== 'plaintext' &&
diff --git a/src/codewhisperer/commands/onAcceptance.ts b/src/codewhisperer/commands/onAcceptance.ts
@@ -94,8 +94,11 @@ export async function onAcceptance(acceptanceEntry: OnRecommendationAcceptanceEn
             completionType: acceptanceEntry.completionType,
             language: languageContext.language,
         })
-        CodeWhispererCodeCoverageTracker.getTracker(languageContext.language, globalStorage).setAcceptedTokens(
-            acceptanceEntry.recommendation
+        const codeRangeAfterFormat = new vscode.Range(start, acceptanceEntry.editor.selection.active)
+        CodeWhispererCodeCoverageTracker.getTracker(languageContext.language, globalStorage)?.countAcceptedTokens(
+            codeRangeAfterFormat,
+            acceptanceEntry.editor.document.getText(codeRangeAfterFormat),
+            acceptanceEntry.editor.document.fileName
         )
     }
 
diff --git a/src/codewhisperer/service/keyStrokeHandler.ts b/src/codewhisperer/service/keyStrokeHandler.ts
@@ -9,11 +9,8 @@ import { DefaultCodeWhispererClient } from '../client/codewhisperer'
 import * as EditorContext from '../util/editorContext'
 import { CodeWhispererConstants } from '../models/constants'
 import { vsCodeState, ConfigurationEntry } from '../models/model'
-import { runtimeLanguageContext } from '../util/runtimeLanguageContext'
 import { getLogger } from '../../shared/logger'
 import { InlineCompletion } from './inlineCompletion'
-import { CodeWhispererCodeCoverageTracker } from '../tracker/codewhispererCodeCoverageTracker'
-import globals from '../../shared/extensionGlobals'
 import { isCloud9 } from '../../shared/extensionUtilities'
 import { RecommendationHandler } from './recommendationHandler'
 
@@ -50,12 +47,6 @@ export class KeyStrokeHandler {
         config: ConfigurationEntry
     ): Promise<void> {
         try {
-            const content = event.contentChanges[0].text
-            const languageContext = runtimeLanguageContext.getLanguageContext(editor.document.languageId)
-            CodeWhispererCodeCoverageTracker.getTracker(
-                languageContext.language,
-                globals.context.globalState
-            ).setTotalTokens(content)
             const changedText = this.getChangedText(event, config.isAutomatedTriggerEnabled, editor)
             if (changedText === '') {
                 return
diff --git a/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts b/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts
@@ -8,88 +8,122 @@ import * as telemetry from '../../shared/telemetry/telemetry'
 import { getLogger } from '../../shared/logger/logger'
 import { CodeWhispererConstants } from '../models/constants'
 import globals from '../../shared/extensionGlobals'
+import { vsCodeState } from '../models/model'
+import { distance } from 'fastest-levenshtein'
+
+interface CodeWhispererToken {
+    range: vscode.Range
+    text: string
+    accepted: number
+}
+
 /**
- * This singleton class is mainly used for calculating the percentage of user modification.
- * The current calculation method is (Levenshtein edit distance / acceptedSuggestion.length).
+ * This singleton class is mainly used for calculating the code written by codeWhisperer
  */
 export class CodeWhispererCodeCoverageTracker {
-    private _acceptedTokens: string[]
-    private _totalTokens: string[]
+    private _acceptedTokens: { [key: string]: CodeWhispererToken[] }
+    private _totalTokens: { [key: string]: number }
     private _timer?: NodeJS.Timer
     private _startTime: number
     private _language: telemetry.CodewhispererLanguage
 
     private constructor(language: telemetry.CodewhispererLanguage, private readonly _globals: vscode.Memento) {
-        this._acceptedTokens = []
-        this._totalTokens = []
+        this._acceptedTokens = {}
+        this._totalTokens = {}
         this._startTime = 0
         this._language = language
     }
 
-    public setAcceptedTokens(recommendation: string) {
-        const terms = this._globals.get<boolean>(CodeWhispererConstants.termsAcceptedKey) || false
-        if (!terms) return
-
-        // generate accepted recoomendation token and stored in collection
-        this._acceptedTokens.push(...recommendation)
-        this._totalTokens.push(...recommendation)
+    public get acceptedTokens(): { [key: string]: CodeWhispererToken[] } {
+        return this._acceptedTokens
     }
-
-    public get AcceptedTokensLength(): number {
-        return this._acceptedTokens.length
+    public get totalTokens(): { [key: string]: number } {
+        return this._totalTokens
     }
 
-    public setTotalTokens(content: string) {
-        if (this._totalTokens.length === 0 && this._timer == undefined) {
-            const currentDate = new globals.clock.Date()
-            this._startTime = currentDate.getTime()
-            this.startTimer()
-        }
-
-        if (content.length <= 2) {
-            this._totalTokens.push(content)
-        } else if (content.length > 2) {
-            this._totalTokens.push(...content)
-        }
+    public countAcceptedTokens(range: vscode.Range, text: string, filename: string) {
+        const terms = this._globals.get<boolean>(CodeWhispererConstants.termsAcceptedKey) || false
+        if (!terms) return
+        // generate accepted recommendation token and stored in collection
+        this.addAcceptedTokens(filename, { range: range, text: text, accepted: text.length })
+        this.addTotalTokens(filename, text.length)
     }
 
     public flush() {
         const terms = this._globals.get<boolean>(CodeWhispererConstants.termsAcceptedKey) || false
         if (!terms) {
-            this._totalTokens = []
-            this._acceptedTokens = []
+            this._totalTokens = {}
+            this._acceptedTokens = {}
             this.closeTimer()
             return
         }
-        this.emitCodeWhispererCodeContribution()
+        try {
+            this.emitCodeWhispererCodeContribution()
+        } catch (error) {
+            getLogger().error(`Encountered ${error} when emitting code contribution metric`)
+        }
+    }
+
+    public updateAcceptedTokensCount(editor: vscode.TextEditor) {
+        const filename = editor.document.fileName
+        if (filename in this._acceptedTokens) {
+            for (let i = 0; i < this._acceptedTokens[filename].length; i++) {
+                const oldText = this._acceptedTokens[filename][i].text
+                const newText = editor.document.getText(this._acceptedTokens[filename][i].range)
+                this._acceptedTokens[filename][i].accepted = this.getUnmodifiedAcceptedTokens(oldText, newText)
+            }
+        }
+    }
+    // With edit distance, complicate usermodification can be considered as simple edit(add, delete, replace),
+    // and thus the unmodified part of recommendation length can be deducted/approximated
+    // ex. (modified > original): originalRecom: foo -> modifiedRecom: fobarbarbaro, distance = 9, delta = 12 - 9 = 3
+    // ex. (modified == original): originalRecom: helloworld -> modifiedRecom: HelloWorld, distance = 2, delta = 10 - 2 = 8
+    // ex. (modified < original): originalRecom: CodeWhisperer -> modifiedRecom: CODE, distance = 12, delta = 13 - 12 = 1
+    public getUnmodifiedAcceptedTokens(origin: string, after: string) {
+        return Math.max(origin.length, after.length) - distance(origin, after)
     }
 
     public emitCodeWhispererCodeContribution() {
-        const totalTokens = this._totalTokens
-        const acceptedTokens = this._acceptedTokens
-        const percentCount = ((acceptedTokens.length / totalTokens.length) * 100).toFixed(2)
+        let totalTokens = 0
+        for (const filename in this._totalTokens) {
+            totalTokens += this._totalTokens[filename]
+        }
+        if (vscode.window.activeTextEditor) {
+            this.updateAcceptedTokensCount(vscode.window.activeTextEditor)
+        }
+        let acceptedTokens = 0
+        for (const filename in this._acceptedTokens) {
+            this._acceptedTokens[filename].forEach(v => {
+                if (filename in this._totalTokens && this._totalTokens[filename] >= v.accepted) {
+                    acceptedTokens += v.accepted
+                }
+            })
+        }
+        const percentCount = ((acceptedTokens / totalTokens) * 100).toFixed(2)
         const percentage = Math.round(parseInt(percentCount))
         telemetry.recordCodewhispererCodePercentage({
-            codewhispererTotalTokens: totalTokens.length ? totalTokens.length : 0,
+            codewhispererTotalTokens: totalTokens,
             codewhispererLanguage: this._language,
-            codewhispererAcceptedTokens: acceptedTokens.length ? acceptedTokens.length : 0,
+            codewhispererAcceptedTokens: acceptedTokens,
             codewhispererPercentage: percentage ? percentage : 0,
         })
     }
 
-    public startTimer() {
-        if (this._timer !== undefined) {
-            return
-        }
+    private tryStartTimer() {
+        if (this._timer !== undefined) return
+        const currentDate = new globals.clock.Date()
+        this._startTime = currentDate.getTime()
         this._timer = setTimeout(() => {
             try {
                 const currentTime = new globals.clock.Date().getTime()
                 const delay: number = CodeWhispererConstants.defaultCheckPeriodMillis
                 const diffTime: number = this._startTime + delay
                 if (diffTime <= currentTime) {
-                    const totalTokens = this._totalTokens
-                    const acceptedTokens = this._acceptedTokens
-                    if (totalTokens.length > 0 && acceptedTokens.length > 0) {
+                    let totalTokens = 0
+                    for (const filename in this._totalTokens) {
+                        totalTokens += this._totalTokens[filename]
+                    }
+                    if (totalTokens > 0) {
                         this.flush()
                     } else {
                         getLogger().debug(
@@ -100,28 +134,67 @@ export class CodeWhispererCodeCoverageTracker {
             } catch (e) {
                 getLogger().verbose(`Exception Thrown from CodeWhispererCodeCoverageTracker: ${e}`)
             } finally {
-                this._totalTokens = []
-                this._acceptedTokens = []
+                this._totalTokens = {}
+                this._acceptedTokens = {}
                 this._startTime = 0
                 this.closeTimer()
             }
         }, CodeWhispererConstants.defaultCheckPeriodMillis)
     }
 
-    public closeTimer() {
+    private closeTimer() {
         if (this._timer !== undefined) {
             clearTimeout(this._timer)
             this._timer = undefined
         }
     }
 
-    public static readonly instances = new Map<telemetry.CodewhispererLanguage, CodeWhispererCodeCoverageTracker>()
-    public static getTracker(
-        language: telemetry.CodewhispererLanguage = 'plaintext',
-        memento: vscode.Memento
-    ): CodeWhispererCodeCoverageTracker {
-        const instance = this.instances.get(language) ?? new this(language, memento)
-        this.instances.set(language, instance)
-        return instance
+    public addAcceptedTokens(filename: string, token: CodeWhispererToken) {
+        if (!(filename in this._acceptedTokens)) {
+            this._acceptedTokens[filename] = []
+        }
+        this._acceptedTokens[filename].push(token)
+    }
+
+    public addTotalTokens(filename: string, count: number) {
+        if (!(filename in this._totalTokens)) {
+            this._totalTokens[filename] = 0
+        }
+        this._totalTokens[filename] += count
+        if (this._totalTokens[filename] < 0) {
+            this._totalTokens[filename] = 0
+        }
+    }
+
+    public countTotalTokens(e: vscode.TextDocumentChangeEvent) {
+        // ignore no contentChanges. ignore contentChanges from other plugins (formatters)
+        // only include contentChanges from user action
+        if (
+            !CodeWhispererConstants.supportedLanguages.includes(e.document.languageId) ||
+            vsCodeState.isCodeWhispererEditing ||
+            e.contentChanges.length !== 1
+        )
+            return
+        const content = e.contentChanges[0]
+        // do not count user tokens if user copies large chunk of code
+        if (content.text.length > 20) return
+        this.tryStartTimer()
+        // deletion events has no text.
+        if (content.text.length === 0) {
+            this.addTotalTokens(e.document.fileName, -content.rangeLength)
+        } else {
+            this.addTotalTokens(e.document.fileName, content.text.length)
+        }
+    }
+
+    public static readonly instances = new Map<string, CodeWhispererCodeCoverageTracker>()
+    public static getTracker(language: string, memento: vscode.Memento): CodeWhispererCodeCoverageTracker | undefined {
+        if (CodeWhispererConstants.supportedLanguages.includes(language)) {
+            const instance =
+                this.instances.get(language) ?? new this(language as telemetry.CodewhispererLanguage, memento)
+            this.instances.set(language, instance)
+            return instance
+        }
+        return undefined
     }
 }
diff --git a/src/test/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts b/src/test/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts

Original file line number	Diff line number	Diff line change
`@@ -94,8 +94,11 @@ export async function onAcceptance(acceptanceEntry: OnRecommendationAcceptanceEn`
`94`	`94`	`completionType: acceptanceEntry.completionType,`
`95`	`95`	`language: languageContext.language,`
`96`	`96`	`})`
`97`		`- CodeWhispererCodeCoverageTracker.getTracker(languageContext.language, globalStorage).setAcceptedTokens(`
`98`		`- acceptanceEntry.recommendation`
	`97`	`+ const codeRangeAfterFormat = new vscode.Range(start, acceptanceEntry.editor.selection.active)`
	`98`	`+ CodeWhispererCodeCoverageTracker.getTracker(languageContext.language, globalStorage)?.countAcceptedTokens(`
	`99`	`+ codeRangeAfterFormat,`
	`100`	`+ acceptanceEntry.editor.document.getText(codeRangeAfterFormat),`
	`101`	`+ acceptanceEntry.editor.document.fileName`
`99`	`102`	`)`
`100`	`103`	`}`
`101`	`104`