Skip to content

Commit 44f1244

Browse files
sandy081sandy081
authored
show only requests aggregated for all models (microsoft#211269)
1 parent 0ed0a9f commit 44f1244

File tree

3 files changed

+41
-223
lines changed

3 files changed

+41
-223
lines changed

‎src/vs/workbench/contrib/chat/common/languageModelStats.ts

Lines changed: 7 additions & 223 deletions
Original file line numberDiff line numberDiff line change
@@ -3,51 +3,21 @@
33
* Licensed under the MIT License. See License.txt in the project root for license information.
44
*--------------------------------------------------------------------------------------------*/
55

6-
import { Emitter, Event } from 'vs/base/common/event';
7-
import { Disposable, DisposableStore, IDisposable } from 'vs/base/common/lifecycle';
6+
import { Emitter } from 'vs/base/common/event';
7+
import { Disposable } from 'vs/base/common/lifecycle';
88
import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
99
import { IStorageService, StorageScope, StorageTarget } from 'vs/platform/storage/common/storage';
10-
import { ExtensionIdentifier, IExtensionManifest } from 'vs/platform/extensions/common/extensions';
11-
import { Extensions, IExtensionFeatureMarkdownAndTableRenderer, IExtensionFeaturesRegistry, IRenderedData, ITableData } from 'vs/workbench/services/extensionManagement/common/extensionFeatures';
12-
import { ILanguageModelsService } from 'vs/workbench/contrib/chat/common/languageModels';
13-
import { getExtensionId } from 'vs/platform/extensionManagement/common/extensionManagementUtil';
14-
import { IMarkdownString, MarkdownString } from 'vs/base/common/htmlContent';
10+
import { ExtensionIdentifier } from 'vs/platform/extensions/common/extensions';
11+
import { Extensions, IExtensionFeaturesManagementService, IExtensionFeaturesRegistry } from 'vs/workbench/services/extensionManagement/common/extensionFeatures';
1512
import { Registry } from 'vs/platform/registry/common/platform';
1613
import { localize } from 'vs/nls';
17-
import { SyncDescriptor } from 'vs/platform/instantiation/common/descriptors';
18-
import { ChatAgentLocation, IChatAgentService } from 'vs/workbench/contrib/chat/common/chatAgents';
19-
20-
export interface ILanguageModelStats {
21-
readonly identifier: string;
22-
readonly extensions: {
23-
readonly extensionId: string;
24-
readonly requestCount: number;
25-
readonly tokenCount: number;
26-
readonly sessionRequestCount: number;
27-
readonly sessionTokenCount: number;
28-
readonly participants: {
29-
readonly id: string;
30-
readonly requestCount: number;
31-
readonly tokenCount: number;
32-
readonly sessionRequestCount: number;
33-
readonly sessionTokenCount: number;
34-
}[];
35-
}[];
36-
}
3714

3815
export const ILanguageModelStatsService = createDecorator<ILanguageModelStatsService>('ILanguageModelStatsService');
3916

4017
export interface ILanguageModelStatsService {
41-
4218
readonly _serviceBrand: undefined;
4319

44-
readonly onDidChangeLanguageMoelStats: Event<string>;
45-
46-
hasAccessedModel(extensionId: string, model: string): boolean;
47-
4820
update(model: string, extensionId: ExtensionIdentifier, agent: string | undefined, tokenCount: number | undefined): Promise<void>;
49-
fetch(model: string): Promise<ILanguageModelStats>;
50-
5121
}
5222

5323
interface LanguageModelStats {
@@ -76,6 +46,7 @@ export class LanguageModelStatsService extends Disposable implements ILanguageMo
7646
private readonly sessionStats = new Map<string, LanguageModelStats>();
7747

7848
constructor(
49+
@IExtensionFeaturesManagementService private readonly extensionFeaturesManagementService: IExtensionFeaturesManagementService,
7950
@IStorageService private readonly _storageService: IStorageService,
8051
) {
8152
super();
@@ -91,35 +62,9 @@ export class LanguageModelStatsService extends Disposable implements ILanguageMo
9162
return this.getAccessExtensions(model).includes(extensionId.toLowerCase());
9263
}
9364

94-
async fetch(model: string): Promise<ILanguageModelStats> {
95-
const globalStats = await this.read(model);
96-
const sessionStats = this.sessionStats.get(model) ?? { extensions: [] };
97-
return {
98-
identifier: model,
99-
extensions: globalStats.extensions.map(extension => {
100-
const sessionExtension = sessionStats.extensions.find(e => e.extensionId === extension.extensionId);
101-
return {
102-
extensionId: extension.extensionId,
103-
requestCount: extension.requestCount,
104-
tokenCount: extension.tokenCount,
105-
sessionRequestCount: sessionExtension?.requestCount ?? 0,
106-
sessionTokenCount: sessionExtension?.tokenCount ?? 0,
107-
participants: extension.participants.map(participant => {
108-
const sessionParticipant = sessionExtension?.participants.find(p => p.id === participant.id);
109-
return {
110-
id: participant.id,
111-
requestCount: participant.requestCount,
112-
tokenCount: participant.tokenCount,
113-
sessionRequestCount: sessionParticipant?.requestCount ?? 0,
114-
sessionTokenCount: sessionParticipant?.tokenCount ?? 0
115-
};
116-
})
117-
};
118-
})
119-
};
120-
}
121-
12265
async update(model: string, extensionId: ExtensionIdentifier, agent: string | undefined, tokenCount: number | undefined): Promise<void> {
66+
await this.extensionFeaturesManagementService.getAccess(extensionId, 'languageModels');
67+
12368
// update model access
12469
this.addAccess(model, extensionId.value);
12570

@@ -215,172 +160,11 @@ export class LanguageModelStatsService extends Disposable implements ILanguageMo
215160
}
216161
}
217162

218-
interface Stats {
219-
requestCount: number;
220-
tokenCount: number;
221-
sessionRequestCount: number;
222-
sessionTokenCount: number;
223-
}
224-
225-
interface ExtensionLanguageModelStats extends Stats {
226-
languageModelId: string;
227-
other: Stats;
228-
participants: Array<Stats & { name: string }>;
229-
}
230-
231-
class LanguageModelFeatureRenderer extends Disposable implements IExtensionFeatureMarkdownAndTableRenderer {
232-
233-
readonly type = 'markdown+table';
234-
235-
constructor(
236-
@ILanguageModelsService private readonly _languageModelsService: ILanguageModelsService,
237-
@ILanguageModelStatsService private readonly _languageModelStatsService: ILanguageModelStatsService,
238-
@IChatAgentService private readonly _chatAgentService: IChatAgentService,
239-
) {
240-
super();
241-
}
242-
243-
shouldRender(manifest: IExtensionManifest): boolean {
244-
if (!!manifest.contributes?.chatParticipants?.length) {
245-
return true;
246-
}
247-
const extensionId = getExtensionId(manifest.publisher, manifest.name);
248-
if (this._languageModelsService.getLanguageModelIds().some(id =>
249-
this._languageModelStatsService.hasAccessedModel(extensionId, id)
250-
|| ExtensionIdentifier.equals(this._languageModelsService.lookupLanguageModel(id)?.extension, extensionId))) {
251-
return true;
252-
}
253-
return false;
254-
}
255-
256-
render(manifest: IExtensionManifest): IRenderedData<Array<IMarkdownString | ITableData>> {
257-
const disposables = new DisposableStore();
258-
const extensionId = getExtensionId(manifest.publisher, manifest.name);
259-
const emitter = disposables.add(new Emitter<Array<IMarkdownString | ITableData>>());
260-
261-
this.fetchAllLanguageModelStats(extensionId).then(({ data, onDidChange, disposable }) => {
262-
disposables.add(disposable);
263-
const renderData = (languageModelStats: ExtensionLanguageModelStats[]) => {
264-
const data: Array<IMarkdownString | ITableData> = [];
265-
for (const stats of languageModelStats) {
266-
if (stats.requestCount > 0) {
267-
const languageModelTitle = new MarkdownString();
268-
languageModelTitle.appendMarkdown(`&nbsp;&nbsp;`);
269-
languageModelTitle.appendMarkdown(`\n\n### ${stats.languageModelId}\n---\n\n`);
270-
data.push(languageModelTitle);
271-
const tableData: ITableData = {
272-
headers: [localize('participant', "Participant"), localize('requests', "Requests"), localize('tokens', "Tokens"), localize('requests session', "Requests (Session)"), localize('tokens session', "Tokens (Session)")],
273-
rows: [
274-
...stats.participants.map(participant => [participant.name, `${participant.requestCount}`, `${participant.tokenCount}`, `${participant.sessionRequestCount}`, `${participant.sessionTokenCount}`]),
275-
stats.other.requestCount > 0 ? [stats.participants.length ? 'Other' : '', `${stats.other.requestCount}`, `${stats.other.tokenCount}`, `${stats.other.sessionRequestCount}`, `${stats.other.sessionTokenCount}`] : [],
276-
stats.participants.length ? ['Total', `${stats.requestCount}`, `${stats.tokenCount}`, `${stats.sessionRequestCount}`, `${stats.sessionTokenCount}`] : [],
277-
]
278-
};
279-
data.push(tableData);
280-
}
281-
}
282-
return data;
283-
};
284-
emitter.fire(renderData(data));
285-
disposables.add(onDidChange(data => emitter.fire(renderData(data))));
286-
});
287-
288-
const data: Array<IMarkdownString | ITableData> = [];
289-
data.push(new MarkdownString().appendMarkdown(`Fetching...`));
290-
291-
return {
292-
data,
293-
onDidChange: emitter.event,
294-
dispose: () => {
295-
disposables.dispose();
296-
}
297-
};
298-
}
299-
300-
private async fetchAllLanguageModelStats(extensionId: string): Promise<{ data: ExtensionLanguageModelStats[]; onDidChange: Event<ExtensionLanguageModelStats[]>; disposable: IDisposable }> {
301-
const disposables = new DisposableStore();
302-
const data: ExtensionLanguageModelStats[] = [];
303-
const emitter = disposables.add(new Emitter<ExtensionLanguageModelStats[]>());
304-
305-
const models = this._languageModelsService.getLanguageModelIds();
306-
for (const model of models) {
307-
data.push(await this.fetchLanguageModelStats(extensionId, model));
308-
}
309-
310-
disposables.add(this._languageModelStatsService.onDidChangeLanguageMoelStats(model => {
311-
this.fetchLanguageModelStats(extensionId, model).then(stats => {
312-
const index = data.findIndex(d => d.languageModelId === model);
313-
if (index !== -1) {
314-
data[index] = stats;
315-
} else {
316-
data.push(stats);
317-
}
318-
emitter.fire(data);
319-
});
320-
}));
321-
322-
return {
323-
data,
324-
onDidChange: emitter.event,
325-
disposable: disposables
326-
};
327-
}
328-
329-
private async fetchLanguageModelStats(extensionId: string, languageModel: string): Promise<ExtensionLanguageModelStats> {
330-
const result: ExtensionLanguageModelStats = {
331-
languageModelId: languageModel,
332-
requestCount: 0,
333-
tokenCount: 0,
334-
sessionRequestCount: 0,
335-
sessionTokenCount: 0,
336-
other: {
337-
requestCount: 0,
338-
tokenCount: 0,
339-
sessionRequestCount: 0,
340-
sessionTokenCount: 0,
341-
},
342-
participants: []
343-
};
344-
const stats = await this._languageModelStatsService.fetch(languageModel);
345-
const extensionStats = stats?.extensions.find(e => ExtensionIdentifier.equals(e.extensionId, extensionId));
346-
if (extensionStats) {
347-
result.requestCount = extensionStats.requestCount;
348-
result.tokenCount = extensionStats.tokenCount;
349-
result.sessionRequestCount = extensionStats.sessionRequestCount;
350-
result.sessionTokenCount = extensionStats.sessionTokenCount;
351-
result.other.requestCount = extensionStats.requestCount;
352-
result.other.tokenCount = extensionStats.tokenCount;
353-
result.other.sessionRequestCount = extensionStats.sessionRequestCount;
354-
result.other.sessionTokenCount = extensionStats.sessionTokenCount;
355-
for (const participant of extensionStats.participants) {
356-
const agent = this._chatAgentService.getAgent(participant.id);
357-
result.requestCount += participant.requestCount;
358-
result.tokenCount += participant.tokenCount;
359-
result.sessionRequestCount += participant.sessionRequestCount;
360-
result.sessionTokenCount += participant.sessionTokenCount;
361-
result.participants.splice(agent?.isDefault ? 0 : result.participants.length, 0, {
362-
name: agent ?
363-
agent?.isDefault ?
364-
agent.locations.includes(ChatAgentLocation.Editor) ? localize('chat editor', "Inline Chat (Editor)") : localize('chat', "Chat")
365-
: `@${agent.name}`
366-
: participant.id,
367-
requestCount: participant.requestCount,
368-
tokenCount: participant.tokenCount,
369-
sessionRequestCount: participant.sessionRequestCount,
370-
sessionTokenCount: participant.sessionTokenCount
371-
});
372-
}
373-
}
374-
return result;
375-
}
376-
}
377-
378163
Registry.as<IExtensionFeaturesRegistry>(Extensions.ExtensionFeaturesRegistry).registerExtensionFeature({
379164
id: 'languageModels',
380165
label: localize('Language Models', "Language Models"),
381166
description: localize('languageModels', "Language models usage statistics of this extension."),
382167
access: {
383168
canToggle: false
384169
},
385-
renderer: new SyncDescriptor(LanguageModelFeatureRenderer),
386170
});

‎src/vs/workbench/contrib/extensions/browser/abstractRuntimeExtensionsEditor.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import { IListAccessibilityProvider } from 'vs/base/browser/ui/list/listWidget';
1212
import { Action, IAction, Separator } from 'vs/base/common/actions';
1313
import { isNonEmptyArray } from 'vs/base/common/arrays';
1414
import { RunOnceScheduler } from 'vs/base/common/async';
15+
import { fromNow } from 'vs/base/common/date';
1516
import { memoize } from 'vs/base/common/decorators';
1617
import { IDisposable, dispose } from 'vs/base/common/lifecycle';
1718
import { Schemas } from 'vs/base/common/network';
@@ -420,6 +421,13 @@ export abstract class AbstractRuntimeExtensionsEditor extends EditorPane {
420421
data.msgContainer.appendChild($('span', undefined, `${feature.label}: `));
421422
data.msgContainer.appendChild($('span', undefined, ...renderLabelWithIcons(`$(${status.severity === Severity.Error ? errorIcon.id : warningIcon.id}) ${status.message}`)));
422423
}
424+
if (accessData?.current) {
425+
const element = $('span', undefined, nls.localize('requests count', "{0} Requests: {1} (Session)", feature.label, accessData.current.count));
426+
const title = nls.localize('requests count title', "Last request was {0}. Overall Requests: {1}", fromNow(accessData.current.lastAccessed, true, true), accessData.totalCount);
427+
data.elementDisposables.push(this._hoverService.setupUpdatableHover(getDefaultHoverDelegate('mouse'), element, title));
428+
429+
data.msgContainer.appendChild(element);
430+
}
423431
}
424432
}
425433

‎src/vs/workbench/contrib/extensions/browser/extensionFeaturesTab.ts

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import { IExtensionService } from 'vs/workbench/services/extensions/common/exten
3737
import { Codicon } from 'vs/base/common/codicons';
3838
import { SyncDescriptor } from 'vs/platform/instantiation/common/descriptors';
3939
import { ResolvedKeybinding } from 'vs/base/common/keybindings';
40+
import { fromNow } from 'vs/base/common/date';
4041

4142
class RuntimeStatusMarkdownRenderer extends Disposable implements IExtensionFeatureMarkdownRenderer {
4243

@@ -45,6 +46,7 @@ class RuntimeStatusMarkdownRenderer extends Disposable implements IExtensionFeat
4546

4647
constructor(
4748
@IExtensionService private readonly extensionService: IExtensionService,
49+
@IExtensionFeaturesManagementService private readonly extensionFeaturesManagementService: IExtensionFeaturesManagementService,
4850
) {
4951
super();
5052
}
@@ -66,6 +68,7 @@ class RuntimeStatusMarkdownRenderer extends Disposable implements IExtensionFeat
6668
emitter.fire(this.getRuntimeStatusData(manifest));
6769
}
6870
}));
71+
disposables.add(this.extensionFeaturesManagementService.onDidChangeAccessData(e => emitter.fire(this.getRuntimeStatusData(manifest))));
6972
return {
7073
onDidChange: emitter.event,
7174
data: this.getRuntimeStatusData(manifest),
@@ -101,6 +104,29 @@ class RuntimeStatusMarkdownRenderer extends Disposable implements IExtensionFeat
101104
}
102105
}
103106
}
107+
const features = Registry.as<IExtensionFeaturesRegistry>(Extensions.ExtensionFeaturesRegistry).getExtensionFeatures();
108+
for (const feature of features) {
109+
const accessData = this.extensionFeaturesManagementService.getAccessData(extensionId, feature.id);
110+
if (accessData) {
111+
data.appendMarkdown(`\n ### ${feature.label}\n\n`);
112+
const status = accessData?.current?.status;
113+
if (status) {
114+
if (status?.severity === Severity.Error) {
115+
data.appendMarkdown(`$(${errorIcon.id}) ${status.message}\n\n`);
116+
}
117+
if (status?.severity === Severity.Warning) {
118+
data.appendMarkdown(`$(${warningIcon.id}) ${status.message}\n\n`);
119+
}
120+
}
121+
if (accessData?.totalCount) {
122+
if (accessData.current) {
123+
data.appendMarkdown(`${localize('last request', "Last Request: `{0}`", fromNow(accessData.current.lastAccessed, true, true))}\n\n`);
124+
data.appendMarkdown(`${localize('requests count session', "Requests (Session) : `{0}`", accessData.current.count)}\n\n`);
125+
}
126+
data.appendMarkdown(`${localize('requests count total', "Requests (Overall): `{0}`", accessData.totalCount)}\n\n`);
127+
}
128+
}
129+
}
104130
return data;
105131
}
106132
}

0 commit comments

Comments
 (0)