Skip to content

Commit 9c14978

Browse files
waldekmastykarzwaldekmastykarz
committed
Extends OpenAITelemetryPlugin with token report. Closes #1308 
1 parent b07d16b commit 9c14978

File tree

2 files changed

+183
-1
lines changed

2 files changed

+183
-1
lines changed

‎DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
using OpenTelemetry.Metrics;
1616
using OpenTelemetry.Resources;
1717
using OpenTelemetry.Trace;
18+
using System.Collections.Concurrent;
1819
using System.Diagnostics;
1920
using System.Diagnostics.Metrics;
2021
using System.Text.Json;
@@ -46,7 +47,7 @@ public sealed class OpenAITelemetryPlugin(
4647
ISet<UrlToWatch> urlsToWatch,
4748
IProxyConfiguration proxyConfiguration,
4849
IConfigurationSection pluginConfigurationSection) :
49-
BasePlugin<OpenAITelemetryPluginConfiguration>(
50+
BaseReportingPlugin<OpenAITelemetryPluginConfiguration>(
5051
httpClient,
5152
logger,
5253
urlsToWatch,
@@ -65,6 +66,7 @@ public sealed class OpenAITelemetryPlugin(
6566
private LanguageModelPricesLoader? _loader;
6667
private MeterProvider? _meterProvider;
6768
private TracerProvider? _tracerProvider;
69+
private readonly ConcurrentDictionary<string, List<OpenAITelemetryPluginReportModelUsageInformation>> _modelUsage = [];
6870

6971
public override string Name => nameof(OpenAITelemetryPlugin);
7072

@@ -189,6 +191,27 @@ public override Task AfterResponseAsync(ProxyResponseArgs e, CancellationToken c
189191
return Task.CompletedTask;
190192
}
191193

194+
public override Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken cancellationToken)
195+
{
196+
Logger.LogTrace("{Method} called", nameof(AfterRecordingStopAsync));
197+
198+
var report = new OpenAITelemetryPluginReport
199+
{
200+
Application = Configuration.Application,
201+
Environment = Configuration.Environment,
202+
Currency = Configuration.Currency,
203+
IncludeCosts = Configuration.IncludeCosts,
204+
ModelUsage = _modelUsage.ToDictionary()
205+
};
206+
207+
StoreReport(report, e);
208+
_modelUsage.Clear();
209+
210+
Logger.LogTrace("Left {Name}", nameof(AfterRecordingStopAsync));
211+
return Task.CompletedTask;
212+
}
213+
214+
192215
private void InitializeOpenTelemetryExporter()
193216
{
194217
Logger.LogTrace("InitializeOpenTelemetryExporter() called");
@@ -811,6 +834,15 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI
811834
.SetTag(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, usage.CompletionTokens)
812835
.SetTag(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS, usage.TotalTokens);
813836

837+
var reportModelUsageInformation = new OpenAITelemetryPluginReportModelUsageInformation
838+
{
839+
Model = response.Model,
840+
PromptTokens = usage.PromptTokens,
841+
CompletionTokens = usage.CompletionTokens
842+
};
843+
var usagePerModel = _modelUsage.GetOrAdd(response.Model, model => []);
844+
usagePerModel.Add(reportModelUsageInformation);
845+
814846
if (!Configuration.IncludeCosts || Configuration.Prices is null)
815847
{
816848
Logger.LogDebug("Cost tracking is disabled or prices data is not available");
@@ -847,6 +879,7 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI
847879
new(SemanticConvention.GEN_AI_REQUEST_MODEL, request.Model),
848880
new(SemanticConvention.GEN_AI_RESPONSE_MODEL, response.Model)
849881
]);
882+
reportModelUsageInformation.Cost = totalCost;
850883
}
851884
else
852885
{

‎DevProxy.Plugins/Inspection/OpenAITelemetryPluginReport.cs‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
using DevProxy.Abstractions.Plugins;
2+
using DevProxy.Abstractions.Utils;
3+
using System.Globalization;
4+
using System.Text;
5+
using System.Text.Json;
6+
using System.Text.Json.Serialization;
7+
8+
namespace DevProxy.Plugins.Inspection;
9+
10+
public class OpenAITelemetryPluginReportModelUsageInformation
11+
{
12+
public required long CompletionTokens { get; init; }
13+
public double Cost { get; set; }
14+
public required string Model { get; init; }
15+
public required long PromptTokens { get; init; }
16+
}
17+
18+
public class OpenAITelemetryPluginReport : IMarkdownReport, IPlainTextReport, IJsonReport
19+
{
20+
public required string Application { get; init; }
21+
public required string Currency { get; init; }
22+
public required string Environment { get; init; }
23+
[JsonIgnore]
24+
public bool IncludeCosts { get; set; }
25+
public required Dictionary<string, List<OpenAITelemetryPluginReportModelUsageInformation>> ModelUsage { get; init; } = [];
26+
27+
public string FileExtension => ".json";
28+
29+
public object ToJson() => JsonSerializer.Serialize(this, ProxyUtils.JsonSerializerOptions);
30+
31+
public string? ToMarkdown()
32+
{
33+
var totalTokens = 0L;
34+
var totalPromptTokens = 0L;
35+
var totalCompletionTokens = 0L;
36+
var totalCost = 0.0;
37+
38+
var sb = new StringBuilder();
39+
_ = sb
40+
.AppendLine(CultureInfo.InvariantCulture, $"# LLM usage report for {Application} in {Environment}")
41+
.AppendLine()
42+
.AppendLine("## Per model usage")
43+
.AppendLine();
44+
45+
foreach (var modelUsage in ModelUsage.OrderBy(m => m.Key))
46+
{
47+
var promptTokens = modelUsage.Value.Sum(u => u.PromptTokens);
48+
var completionTokens = modelUsage.Value.Sum(u => u.CompletionTokens);
49+
var tokens = promptTokens + completionTokens;
50+
51+
totalPromptTokens += promptTokens;
52+
totalCompletionTokens += completionTokens;
53+
totalTokens += tokens;
54+
55+
_ = sb.AppendLine("<details>")
56+
.AppendLine(CultureInfo.InvariantCulture, $"<summary><h3>{modelUsage.Key}</h3></summary>")
57+
.AppendLine()
58+
.AppendLine("Metric|Value")
59+
.AppendLine(":-----|----:")
60+
.AppendLine(CultureInfo.InvariantCulture, $"Prompt Tokens|{promptTokens}")
61+
.AppendLine(CultureInfo.InvariantCulture, $"Completion Tokens|{completionTokens}")
62+
.AppendLine(CultureInfo.InvariantCulture, $"Total Tokens|{tokens}");
63+
64+
if (IncludeCosts)
65+
{
66+
var cost = modelUsage.Value.Sum(u => u.Cost);
67+
totalCost += cost;
68+
_ = sb.AppendLine(CultureInfo.InvariantCulture, $"Total Cost|{FormatCost(cost, Currency)}");
69+
}
70+
71+
_ = sb.AppendLine("</details>");
72+
}
73+
74+
_ = sb
75+
.AppendLine()
76+
.AppendLine("## Totals")
77+
.AppendLine()
78+
.AppendLine("Metric|Value")
79+
.AppendLine(":-----|----:")
80+
.AppendLine(CultureInfo.InvariantCulture, $"Prompt Tokens|{totalPromptTokens}")
81+
.AppendLine(CultureInfo.InvariantCulture, $"Completion Tokens|{totalCompletionTokens}")
82+
.AppendLine(CultureInfo.InvariantCulture, $"Total Tokens|{totalTokens}");
83+
84+
if (IncludeCosts)
85+
{
86+
_ = sb.AppendLine(CultureInfo.InvariantCulture, $"Total Cost|{FormatCost(totalCost, Currency)}");
87+
}
88+
return sb.ToString();
89+
}
90+
91+
public string? ToPlainText()
92+
{
93+
var totalTokens = 0L;
94+
var totalPromptTokens = 0L;
95+
var totalCompletionTokens = 0L;
96+
var totalCost = 0.0;
97+
98+
var sb = new StringBuilder();
99+
_ = sb
100+
.AppendLine(CultureInfo.InvariantCulture, $"LLM USAGE REPORT FOR {Application} IN {Environment}")
101+
.AppendLine()
102+
.AppendLine("PER MODEL USAGE")
103+
.AppendLine();
104+
105+
foreach (var modelUsage in ModelUsage.OrderBy(m => m.Key))
106+
{
107+
var promptTokens = modelUsage.Value.Sum(u => u.PromptTokens);
108+
var completionTokens = modelUsage.Value.Sum(u => u.CompletionTokens);
109+
var tokens = promptTokens + completionTokens;
110+
111+
totalPromptTokens += promptTokens;
112+
totalCompletionTokens += completionTokens;
113+
totalTokens += tokens;
114+
115+
_ = sb.AppendLine(CultureInfo.InvariantCulture, $"MODEL: {modelUsage.Key}")
116+
.AppendLine()
117+
.AppendLine(CultureInfo.InvariantCulture, $"Prompt Tokens: {promptTokens}")
118+
.AppendLine(CultureInfo.InvariantCulture, $"Completion Tokens: {completionTokens}")
119+
.AppendLine(CultureInfo.InvariantCulture, $"Total Tokens: {tokens}");
120+
121+
if (IncludeCosts)
122+
{
123+
var cost = modelUsage.Value.Sum(u => u.Cost);
124+
totalCost += cost;
125+
_ = sb.AppendLine(CultureInfo.InvariantCulture, $"Total Cost: {FormatCost(cost, Currency)}");
126+
}
127+
128+
_ = sb.AppendLine();
129+
}
130+
131+
_ = sb
132+
.AppendLine("TOTALS")
133+
.AppendLine()
134+
.AppendLine(CultureInfo.InvariantCulture, $"Prompt Tokens: {totalPromptTokens}")
135+
.AppendLine(CultureInfo.InvariantCulture, $"Completion Tokens: {totalCompletionTokens}")
136+
.AppendLine(CultureInfo.InvariantCulture, $"Total Tokens: {totalTokens}");
137+
138+
if (IncludeCosts)
139+
{
140+
_ = sb.AppendLine(CultureInfo.InvariantCulture, $"Total Cost: {FormatCost(totalCost, Currency)}");
141+
}
142+
return sb.ToString();
143+
}
144+
145+
private static string FormatCost(double cost, string currency)
146+
{
147+
return $"{cost.ToString("#,##0.00########", CultureInfo.InvariantCulture)} {currency}";
148+
}
149+
}

0 commit comments

Comments
 (0)