Skip to content

Commit ecd0405

Browse files
rzhao271rzhao271
authored
feat: time the different search providers (microsoft#252700)
1 parent 8f023e4 commit ecd0405

File tree

3 files changed

+59
-12
lines changed

3 files changed

+59
-12
lines changed

‎src/vs/workbench/contrib/preferences/browser/preferencesSearch.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import { IAiSettingsSearchService } from '../../../services/aiSettingsSearch/com
2020
import { IWorkbenchExtensionEnablementService } from '../../../services/extensionManagement/common/extensionManagement.js';
2121
import { IGroupFilter, ISearchResult, ISetting, ISettingMatch, ISettingMatcher, ISettingsEditorModel, ISettingsGroup, SettingKeyMatchTypes, SettingMatchType } from '../../../services/preferences/common/preferences.js';
2222
import { nullRange } from '../../../services/preferences/common/preferencesModels.js';
23-
import { IAiSearchProvider, IPreferencesSearchService, IRemoteSearchProvider, ISearchProvider, IWorkbenchSettingsConfiguration } from '../common/preferences.js';
23+
import { EMBEDDINGS_ONLY_SEARCH_PROVIDER_NAME, EMBEDDINGS_SEARCH_PROVIDER_NAME, IAiSearchProvider, IPreferencesSearchService, IRemoteSearchProvider, ISearchProvider, IWorkbenchSettingsConfiguration, LLM_RANKED_SEARCH_PROVIDER_NAME, STRING_MATCH_SEARCH_PROVIDER_NAME, TF_IDF_SEARCH_PROVIDER_NAME } from '../common/preferences.js';
2424

2525
export interface IEndpointDetails {
2626
urlBase?: string;
@@ -134,7 +134,7 @@ export class LocalSearchProvider implements ISearchProvider {
134134
const alwaysAllowedMatchTypes = SettingMatchType.DescriptionOrValueMatch | SettingMatchType.LanguageTagSettingMatch;
135135
const filteredMatches = filterMatches
136136
.filter(m => (m.matchType & topKeyMatchType) || (m.matchType & alwaysAllowedMatchTypes) || m.matchType === SettingMatchType.ExactMatch)
137-
.map(m => ({ ...m, providerName: 'local' }));
137+
.map(m => ({ ...m, providerName: STRING_MATCH_SEARCH_PROVIDER_NAME }));
138138
return Promise.resolve({
139139
filterMatches: filteredMatches,
140140
exactMatch: filteredMatches.some(m => m.matchType === SettingMatchType.ExactMatch)
@@ -441,7 +441,7 @@ class EmbeddingsSearchProvider implements IRemoteSearchProvider {
441441
return [];
442442
}
443443

444-
const providerName = this._excludeSelectionStep ? 'embeddingsOnly' : 'embeddingsFull';
444+
const providerName = this._excludeSelectionStep ? EMBEDDINGS_ONLY_SEARCH_PROVIDER_NAME : EMBEDDINGS_SEARCH_PROVIDER_NAME;
445445
for (const settingKey of settings) {
446446
if (filterMatches.length === EmbeddingsSearchProvider.EMBEDDINGS_SETTINGS_SEARCH_MAX_PICKS) {
447447
break;
@@ -550,7 +550,7 @@ class TfIdfSearchProvider implements IRemoteSearchProvider {
550550
matchType: SettingMatchType.RemoteMatch,
551551
keyMatchScore: 0,
552552
score: info.score,
553-
providerName: 'tfIdf'
553+
providerName: TF_IDF_SEARCH_PROVIDER_NAME
554554
});
555555
}
556556

@@ -639,7 +639,7 @@ class AiSearchProvider implements IAiSearchProvider {
639639
matchType: SettingMatchType.RemoteMatch,
640640
keyMatchScore: 0,
641641
score: 0, // the results are sorted upstream.
642-
providerName: 'llmRanked'
642+
providerName: LLM_RANKED_SEARCH_PROVIDER_NAME
643643
});
644644
}
645645

‎src/vs/workbench/contrib/preferences/browser/settingsEditor2.ts

Lines changed: 47 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import { Iterable } from '../../../../base/common/iterator.js';
2323
import { KeyCode } from '../../../../base/common/keyCodes.js';
2424
import { Disposable, DisposableStore, dispose, type IDisposable, MutableDisposable } from '../../../../base/common/lifecycle.js';
2525
import * as platform from '../../../../base/common/platform.js';
26+
import { StopWatch } from '../../../../base/common/stopwatch.js';
2627
import { ThemeIcon } from '../../../../base/common/themables.js';
2728
import { URI } from '../../../../base/common/uri.js';
2829
import { ILanguageService } from '../../../../editor/common/languages/language.js';
@@ -58,7 +59,7 @@ import { nullRange, Settings2EditorModel } from '../../../services/preferences/c
5859
import { IUserDataProfileService } from '../../../services/userDataProfile/common/userDataProfile.js';
5960
import { IUserDataSyncWorkbenchService } from '../../../services/userDataSync/common/userDataSync.js';
6061
import { SuggestEnabledInput } from '../../codeEditor/browser/suggestEnabledInput/suggestEnabledInput.js';
61-
import { CONTEXT_SETTINGS_EDITOR, CONTEXT_SETTINGS_ROW_FOCUS, CONTEXT_SETTINGS_SEARCH_FOCUS, CONTEXT_TOC_ROW_FOCUS, ENABLE_LANGUAGE_FILTER, EXTENSION_FETCH_TIMEOUT_MS, EXTENSION_SETTING_TAG, FEATURE_SETTING_TAG, getExperimentalExtensionToggleData, ID_SETTING_TAG, IPreferencesSearchService, ISearchProvider, LANGUAGE_SETTING_TAG, MODIFIED_SETTING_TAG, POLICY_SETTING_TAG, REQUIRE_TRUSTED_WORKSPACE_SETTING_TAG, SETTINGS_EDITOR_COMMAND_CLEAR_SEARCH_RESULTS, SETTINGS_EDITOR_COMMAND_SHOW_AI_RESULTS, SETTINGS_EDITOR_COMMAND_SUGGEST_FILTERS, WorkbenchSettingsEditorSettings, WORKSPACE_TRUST_SETTING_TAG } from '../common/preferences.js';
62+
import { CONTEXT_SETTINGS_EDITOR, CONTEXT_SETTINGS_ROW_FOCUS, CONTEXT_SETTINGS_SEARCH_FOCUS, CONTEXT_TOC_ROW_FOCUS, EMBEDDINGS_SEARCH_PROVIDER_NAME, ENABLE_LANGUAGE_FILTER, EXTENSION_FETCH_TIMEOUT_MS, EXTENSION_SETTING_TAG, FEATURE_SETTING_TAG, FILTER_MODEL_SEARCH_PROVIDER_NAME, getExperimentalExtensionToggleData, ID_SETTING_TAG, IPreferencesSearchService, ISearchProvider, LANGUAGE_SETTING_TAG, LLM_RANKED_SEARCH_PROVIDER_NAME, MODIFIED_SETTING_TAG, POLICY_SETTING_TAG, REQUIRE_TRUSTED_WORKSPACE_SETTING_TAG, SETTINGS_EDITOR_COMMAND_CLEAR_SEARCH_RESULTS, SETTINGS_EDITOR_COMMAND_SHOW_AI_RESULTS, SETTINGS_EDITOR_COMMAND_SUGGEST_FILTERS, STRING_MATCH_SEARCH_PROVIDER_NAME, TF_IDF_SEARCH_PROVIDER_NAME, WorkbenchSettingsEditorSettings, WORKSPACE_TRUST_SETTING_TAG } from '../common/preferences.js';
6263
import { settingsHeaderBorder, settingsSashBorder, settingsTextInputBorder } from '../common/settingsEditorColorRegistry.js';
6364
import './media/settingsEditor2.css';
6465
import { preferencesAiResultsIcon, preferencesClearInputIcon, preferencesFilterIcon } from './preferencesIcons.js';
@@ -191,6 +192,8 @@ export class SettingsEditor2 extends EditorPane {
191192
private searchInProgress: CancellationTokenSource | null = null;
192193
private aiSearchPromise: CancelablePromise<void> | null = null;
193194

195+
private stopWatch: StopWatch;
196+
194197
private showAiResultsAction: Action | null = null;
195198

196199
private searchInputDelayer: Delayer<void>;
@@ -276,6 +279,7 @@ export class SettingsEditor2 extends EditorPane {
276279
this.settingRowFocused = CONTEXT_SETTINGS_ROW_FOCUS.bindTo(contextKeyService);
277280

278281
this.scheduledRefreshes = new Map<string, DisposableStore>();
282+
this.stopWatch = new StopWatch(false);
279283

280284
this.editorMemento = this.getEditorMemento<ISettingsEditor2State>(editorGroupService, textResourceConfigurationService, SETTINGS_EDITOR_STATE_KEY);
281285

@@ -1778,7 +1782,7 @@ export class SettingsEditor2 extends EditorPane {
17781782
matchType: SettingMatchType.None,
17791783
keyMatchScore: 0,
17801784
score: 0,
1781-
providerName: 'filterModel'
1785+
providerName: FILTER_MODEL_SEARCH_PROVIDER_NAME
17821786
});
17831787
}
17841788
}
@@ -1857,15 +1861,15 @@ export class SettingsEditor2 extends EditorPane {
18571861

18581862
private doLocalSearch(query: string, token: CancellationToken): Promise<ISearchResult | null> {
18591863
const localSearchProvider = this.preferencesSearchService.getLocalSearchProvider(query);
1860-
return this.searchWithProvider(SearchResultIdx.Local, localSearchProvider, token);
1864+
return this.searchWithProvider(SearchResultIdx.Local, localSearchProvider, STRING_MATCH_SEARCH_PROVIDER_NAME, token);
18611865
}
18621866

18631867
private doRemoteSearch(query: string, token: CancellationToken): Promise<ISearchResult | null> {
18641868
const remoteSearchProvider = this.preferencesSearchService.getRemoteSearchProvider(query);
18651869
if (!remoteSearchProvider) {
18661870
return Promise.resolve(null);
18671871
}
1868-
return this.searchWithProvider(SearchResultIdx.Remote, remoteSearchProvider, token);
1872+
return this.searchWithProvider(SearchResultIdx.Remote, remoteSearchProvider, TF_IDF_SEARCH_PROVIDER_NAME, token);
18691873
}
18701874

18711875
private async doAiSearch(query: string, token: CancellationToken): Promise<ISearchResult | null> {
@@ -1874,7 +1878,7 @@ export class SettingsEditor2 extends EditorPane {
18741878
return null;
18751879
}
18761880

1877-
const embeddingsResults = await this.searchWithProvider(SearchResultIdx.Embeddings, aiSearchProvider, token);
1881+
const embeddingsResults = await this.searchWithProvider(SearchResultIdx.Embeddings, aiSearchProvider, EMBEDDINGS_SEARCH_PROVIDER_NAME, token);
18781882
if (!embeddingsResults || token.isCancellationRequested) {
18791883
return null;
18801884
}
@@ -1896,26 +1900,62 @@ export class SettingsEditor2 extends EditorPane {
18961900
return null;
18971901
}
18981902

1903+
this.stopWatch.reset();
18991904
const result = await aiSearchProvider.getLLMRankedResults(token);
1900-
if (!result || token.isCancellationRequested) {
1905+
this.stopWatch.stop();
1906+
1907+
if (token.isCancellationRequested) {
19011908
return null;
19021909
}
19031910

1911+
// Only log the elapsed time if there are actual results.
1912+
if (result && result.filterMatches.length > 0) {
1913+
const elapsed = this.stopWatch.elapsed();
1914+
this.logSearchPerformance(LLM_RANKED_SEARCH_PROVIDER_NAME, elapsed);
1915+
}
1916+
19041917
this.searchResultModel!.setResult(SearchResultIdx.AiSelected, result);
19051918
return result;
19061919
}
19071920

1908-
private async searchWithProvider(type: SearchResultIdx, searchProvider: ISearchProvider, token: CancellationToken): Promise<ISearchResult | null> {
1921+
private async searchWithProvider(type: SearchResultIdx, searchProvider: ISearchProvider, providerName: string, token: CancellationToken): Promise<ISearchResult | null> {
1922+
this.stopWatch.reset();
19091923
const result = await this._searchPreferencesModel(this.defaultSettingsEditorModel, searchProvider, token);
1924+
this.stopWatch.stop();
1925+
19101926
if (token.isCancellationRequested) {
19111927
// Handle cancellation like this because cancellation is lost inside the search provider due to async/await
19121928
return null;
19131929
}
1930+
1931+
// Only log the elapsed time if there are actual results.
1932+
if (result && result.filterMatches.length > 0) {
1933+
const elapsed = this.stopWatch.elapsed();
1934+
this.logSearchPerformance(providerName, elapsed);
1935+
}
1936+
19141937
this.searchResultModel ??= this.instantiationService.createInstance(SearchResultModel, this.viewState, this.settingsOrderByTocIndex, this.workspaceTrustManagementService.isWorkspaceTrusted());
19151938
this.searchResultModel.setResult(type, result);
19161939
return result;
19171940
}
19181941

1942+
private logSearchPerformance(providerName: string, elapsed: number): void {
1943+
type SettingsEditorSearchPerformanceEvent = {
1944+
providerName: string | undefined;
1945+
elapsedMs: number;
1946+
};
1947+
type SettingsEditorSearchPerformanceClassification = {
1948+
providerName: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the search provider, if applicable.' };
1949+
elapsedMs: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The time taken to perform the search, in milliseconds.' };
1950+
owner: 'rzhao271';
1951+
comment: 'Event emitted when the Settings editor calls a search provider to search for a setting';
1952+
};
1953+
this.telemetryService.publicLog2<SettingsEditorSearchPerformanceEvent, SettingsEditorSearchPerformanceClassification>('settingsEditor.searchPerformance', {
1954+
providerName,
1955+
elapsedMs: elapsed,
1956+
});
1957+
}
1958+
19191959
private renderResultCountMessages(showAiResultsMessage: boolean) {
19201960
if (!this.currentSettingsModel) {
19211961
return;

‎src/vs/workbench/contrib/preferences/common/preferences.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,13 @@ export const ENABLE_LANGUAGE_FILTER = true;
110110
export const ENABLE_EXTENSION_TOGGLE_SETTINGS = true;
111111
export const EXTENSION_FETCH_TIMEOUT_MS = 1000;
112112

113+
export const STRING_MATCH_SEARCH_PROVIDER_NAME = 'local';
114+
export const TF_IDF_SEARCH_PROVIDER_NAME = 'tfIdf';
115+
export const FILTER_MODEL_SEARCH_PROVIDER_NAME = 'filterModel';
116+
export const EMBEDDINGS_ONLY_SEARCH_PROVIDER_NAME = 'embeddingsOnly';
117+
export const EMBEDDINGS_SEARCH_PROVIDER_NAME = 'embeddingsFull';
118+
export const LLM_RANKED_SEARCH_PROVIDER_NAME = 'llmRanked';
119+
113120
export enum WorkbenchSettingsEditorSettings {
114121
ShowAISearchToggle = 'workbench.settings.showAISearchToggle',
115122
EnableNaturalLanguageSearch = 'workbench.settings.enableNaturalLanguageSearch',

0 commit comments

Comments
 (0)