Skip to content

Commit 3ce9cb1

Browse files
waldekmastykarzwaldekmastykarz
authored
Adds logging LLM usage information using OpenTelemetry. Closes #1067 (#1167)
1 parent 162c90e commit 3ce9cb1

File tree

11 files changed

+1460
-10
lines changed

11 files changed

+1460
-10
lines changed

‎dev-proxy-abstractions/LanguageModel/OpenAIModels.cs‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,3 +143,106 @@ public class OpenAIChatCompletionResponseChoiceMessage
143143
public string Content { get; set; } = string.Empty;
144144
public string Role { get; set; } = string.Empty;
145145
}
146+
147+
public class OpenAIAudioRequest : OpenAIRequest
148+
{
149+
public string File { get; set; } = string.Empty;
150+
[JsonPropertyName("response_format")]
151+
public string? ResponseFormat { get; set; }
152+
public string? Prompt { get; set; }
153+
public string? Language { get; set; }
154+
}
155+
156+
public class OpenAIAudioSpeechRequest : OpenAIRequest
157+
{
158+
public string Input { get; set; } = string.Empty;
159+
public string Voice { get; set; } = string.Empty;
160+
[JsonPropertyName("response_format")]
161+
public string? ResponseFormat { get; set; }
162+
public double? Speed { get; set; }
163+
}
164+
165+
public class OpenAIAudioTranscriptionResponse : OpenAIResponse
166+
{
167+
public string Text { get; set; } = string.Empty;
168+
public override string? Response => Text;
169+
}
170+
171+
public class OpenAIEmbeddingRequest : OpenAIRequest
172+
{
173+
public string? Input { get; set; }
174+
[JsonPropertyName("encoding_format")]
175+
public string? EncodingFormat { get; set; }
176+
public int? Dimensions { get; set; }
177+
}
178+
179+
public class OpenAIEmbeddingResponse : OpenAIResponse
180+
{
181+
public OpenAIEmbeddingData[]? Data { get; set; }
182+
public override string? Response => null; // Embeddings don't have a text response
183+
}
184+
185+
public class OpenAIEmbeddingData
186+
{
187+
public float[]? Embedding { get; set; }
188+
public int Index { get; set; }
189+
public string? Object { get; set; }
190+
}
191+
192+
public class OpenAIFineTuneRequest : OpenAIRequest
193+
{
194+
[JsonPropertyName("training_file")]
195+
public string TrainingFile { get; set; } = string.Empty;
196+
[JsonPropertyName("validation_file")]
197+
public string? ValidationFile { get; set; }
198+
public int? Epochs { get; set; }
199+
[JsonPropertyName("batch_size")]
200+
public int? BatchSize { get; set; }
201+
[JsonPropertyName("learning_rate_multiplier")]
202+
public double? LearningRateMultiplier { get; set; }
203+
public string? Suffix { get; set; }
204+
}
205+
206+
public class OpenAIFineTuneResponse : OpenAIResponse
207+
{
208+
[JsonPropertyName("fine_tuned_model")]
209+
public string? FineTunedModel { get; set; }
210+
public string Status { get; set; } = string.Empty;
211+
public string? Organization { get; set; }
212+
public long CreatedAt { get; set; }
213+
public long UpdatedAt { get; set; }
214+
[JsonPropertyName("training_file")]
215+
public string TrainingFile { get; set; } = string.Empty;
216+
[JsonPropertyName("validation_file")]
217+
public string? ValidationFile { get; set; }
218+
[JsonPropertyName("result_files")]
219+
public object[]? ResultFiles { get; set; }
220+
public override string? Response => FineTunedModel;
221+
}
222+
223+
public class OpenAIImageRequest : OpenAIRequest
224+
{
225+
public string Prompt { get; set; } = string.Empty;
226+
public int? N { get; set; }
227+
public string? Size { get; set; }
228+
[JsonPropertyName("response_format")]
229+
public string? ResponseFormat { get; set; }
230+
public string? User { get; set; }
231+
public string? Quality { get; set; }
232+
public string? Style { get; set; }
233+
}
234+
235+
public class OpenAIImageResponse : OpenAIResponse
236+
{
237+
public OpenAIImageData[]? Data { get; set; }
238+
public override string? Response => null; // Image responses don't have a text response
239+
}
240+
241+
public class OpenAIImageData
242+
{
243+
public string? Url { get; set; }
244+
[JsonPropertyName("b64_json")]
245+
public string? Base64Json { get; set; }
246+
[JsonPropertyName("revised_prompt")]
247+
public string? RevisedPrompt { get; set; }
248+
}

‎dev-proxy-abstractions/LanguageModel/PricesData.cs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System.Diagnostics;
6+
7+
namespace DevProxy.Abstractions.LanguageModel;
8+
9+
public class PricesData: Dictionary<string, ModelPrices>
10+
{
11+
public bool TryGetModelPrices(string modelName, out ModelPrices? prices)
12+
{
13+
prices = new ModelPrices();
14+
15+
if (string.IsNullOrEmpty(modelName))
16+
{
17+
return false;
18+
}
19+
20+
// Try exact match first
21+
if (TryGetValue(modelName, out prices))
22+
{
23+
return true;
24+
}
25+
26+
// Try to find a matching prefix
27+
// This handles cases like "gpt-4-turbo-2024-04-09" matching with "gpt-4"
28+
var matchingModel = Keys
29+
.Where(k => modelName.StartsWith(k, StringComparison.OrdinalIgnoreCase))
30+
.OrderByDescending(k => k.Length)
31+
.FirstOrDefault();
32+
33+
if (matchingModel != null && TryGetValue(matchingModel, out prices))
34+
{
35+
return true;
36+
}
37+
38+
return false;
39+
}
40+
41+
public (double Input, double Output) CalculateCost(string modelName, long inputTokens, long outputTokens)
42+
{
43+
if (!TryGetModelPrices(modelName, out var prices))
44+
{
45+
return (0, 0);
46+
}
47+
48+
Debug.Assert(prices != null, "Prices data should not be null here.");
49+
50+
// Prices in the data are per 1M tokens
51+
var inputCost = prices.Input * (inputTokens / 1_000_000.0);
52+
var outputCost = prices.Output * (outputTokens / 1_000_000.0);
53+
54+
return (inputCost, outputCost);
55+
}
56+
}
57+
58+
public class ModelPrices
59+
{
60+
public double Input { get; set; }
61+
public double Output { get; set; }
62+
}

‎dev-proxy-abstractions/OpenTelemetry/SemanticConvention.cs‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
namespace DevProxy.Abstractions.OpenTelemetry;
6+
7+
public static class SemanticConvention
8+
{
9+
// GenAI General
10+
public const string GEN_AI_ENDPOINT = "gen_ai.endpoint";
11+
public const string GEN_AI_SYSTEM = "gen_ai.system";
12+
public const string GEN_AI_ENVIRONMENT = "gen_ai.environment";
13+
public const string GEN_AI_APPLICATION_NAME = "gen_ai.application_name";
14+
public const string GEN_AI_OPERATION = "gen_ai.type";
15+
public const string GEN_AI_OPERATION_NAME = "gen_ai.operation.name";
16+
public const string GEN_AI_HUB_OWNER = "gen_ai.hub.owner";
17+
public const string GEN_AI_HUB_REPO = "gen_ai.hub.repo";
18+
public const string GEN_AI_RETRIEVAL_SOURCE = "gen_ai.retrieval.source";
19+
public const string GEN_AI_REQUESTS = "gen_ai.total.requests";
20+
21+
// GenAI Request
22+
public const string GEN_AI_REQUEST_MODEL = "gen_ai.request.model";
23+
public const string GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature";
24+
public const string GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p";
25+
public const string GEN_AI_REQUEST_TOP_K = "gen_ai.request.top_k";
26+
public const string GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens";
27+
public const string GEN_AI_REQUEST_IS_STREAM = "gen_ai.request.is_stream";
28+
public const string GEN_AI_REQUEST_USER = "gen_ai.request.user";
29+
public const string GEN_AI_REQUEST_SEED = "gen_ai.request.seed";
30+
public const string GEN_AI_REQUEST_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty";
31+
public const string GEN_AI_REQUEST_PRESENCE_PENALTY = "gen_ai.request.presence_penalty";
32+
public const string GEN_AI_REQUEST_ENCODING_FORMATS = "gen_ai.request.embedding_format";
33+
public const string GEN_AI_REQUEST_EMBEDDING_DIMENSION = "gen_ai.request.embedding_dimension";
34+
public const string GEN_AI_REQUEST_TOOL_CHOICE = "gen_ai.request.tool_choice";
35+
public const string GEN_AI_REQUEST_AUDIO_VOICE = "gen_ai.request.audio_voice";
36+
public const string GEN_AI_REQUEST_AUDIO_RESPONSE_FORMAT = "gen_ai.request.audio_response_format";
37+
public const string GEN_AI_REQUEST_AUDIO_SPEED = "gen_ai.request.audio_speed";
38+
public const string GEN_AI_REQUEST_FINETUNE_STATUS = "gen_ai.request.fine_tune_status";
39+
public const string GEN_AI_REQUEST_FINETUNE_MODEL_SUFFIX = "gen_ai.request.fine_tune_model_suffix";
40+
public const string GEN_AI_REQUEST_FINETUNE_MODEL_EPOCHS = "gen_ai.request.fine_tune_n_epochs";
41+
public const string GEN_AI_REQUEST_FINETUNE_MODEL_LRM = "gen_ai.request.learning_rate_multiplier";
42+
public const string GEN_AI_REQUEST_FINETUNE_BATCH_SIZE = "gen_ai.request.fine_tune_batch_size";
43+
public const string GEN_AI_REQUEST_VALIDATION_FILE = "gen_ai.request.validation_file";
44+
public const string GEN_AI_REQUEST_TRAINING_FILE = "gen_ai.request.training_file";
45+
public const string GEN_AI_REQUEST_IMAGE_SIZE = "gen_ai.request.image_size";
46+
public const string GEN_AI_REQUEST_IMAGE_QUALITY = "gen_ai.request.image_quality";
47+
public const string GEN_AI_REQUEST_IMAGE_STYLE = "gen_ai.request.image_style";
48+
49+
// GenAI Usage
50+
public const string GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens";
51+
public const string GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens";
52+
// OpenLIT
53+
public const string GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens";
54+
public const string GEN_AI_USAGE_COST = "gen_ai.usage.cost";
55+
public const string GEN_AI_USAGE_TOTAL_COST = "gen_ai.usage.total_cost";
56+
57+
// GenAI Response
58+
public const string GEN_AI_RESPONSE_ID = "gen_ai.response.id";
59+
public const string GEN_AI_RESPONSE_MODEL = "gen_ai.response.model";
60+
public const string GEN_AI_RESPONSE_FINISH_REASON = "gen_ai.response.finish_reason";
61+
public const string GEN_AI_RESPONSE_IMAGE = "gen_ai.response.image";
62+
public const string GEN_AI_RESPONSE_IMAGE_SIZE = "gen_ai.request.image_size";
63+
public const string GEN_AI_RESPONSE_IMAGE_QUALITY = "gen_ai.request.image_quality";
64+
public const string GEN_AI_RESPONSE_IMAGE_STYLE = "gen_ai.request.image_style";
65+
66+
// GenAI Content
67+
public const string GEN_AI_CONTENT_PROMPT = "gen_ai.content.prompt";
68+
public const string GEN_AI_CONTENT_COMPLETION = "gen_ai.completion";
69+
public const string GEN_AI_CONTENT_REVISED_PROMPT = "gen_ai.content.revised_prompt";
70+
71+
// Operation Types
72+
public const string GEN_AI_OPERATION_TYPE_CHAT = "chat";
73+
public const string GEN_AI_OPERATION_TYPE_EMBEDDING = "embedding";
74+
public const string GEN_AI_OPERATION_TYPE_IMAGE = "image";
75+
public const string GEN_AI_OPERATION_TYPE_AUDIO = "audio";
76+
public const string GEN_AI_OPERATION_TYPE_FINETUNING = "fine_tuning";
77+
public const string GEN_AI_OPERATION_TYPE_VECTORDB = "vectordb";
78+
public const string GEN_AI_OPERATION_TYPE_FRAMEWORK = "framework";
79+
80+
// Metrics
81+
public const string GEN_AI_METRIC_CLIENT_TOKEN_USAGE = "gen_ai.client.token.usage";
82+
public const string GEN_AI_TOKEN_TYPE = "gen_ai.token.type";
83+
public const string GEN_AI_TOKEN_TYPE_INPUT = "input";
84+
public const string GEN_AI_TOKEN_TYPE_OUTPUT = "output";
85+
public const string GEN_AI_METRIC_CLIENT_OPERATION_DURATION = "gen_ai.client.operation.duration";
86+
}

‎dev-proxy-plugins/Inspection/LanguageModelPricingLoader.cs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using DevProxy.Abstractions;
6+
using DevProxy.Abstractions.LanguageModel;
7+
using Microsoft.Extensions.Logging;
8+
using System.Text.Json;
9+
10+
namespace DevProxy.Plugins.Inspection;
11+
12+
internal class LanguageModelPricesLoader(ILogger logger, LanguageModelPricesPluginConfiguration configuration, bool validateSchemas) : BaseLoader(logger, validateSchemas)
13+
{
14+
private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger));
15+
private readonly LanguageModelPricesPluginConfiguration _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
16+
protected override string FilePath => Path.Combine(Directory.GetCurrentDirectory(), _configuration.PricesFile ?? "");
17+
18+
protected override void LoadData(string fileContents)
19+
{
20+
try
21+
{
22+
// we need to deserialize manually because standard deserialization
23+
// doesn't support nested dictionaries
24+
using JsonDocument document = JsonDocument.Parse(fileContents);
25+
26+
if (document.RootElement.TryGetProperty("prices", out JsonElement pricesElement))
27+
{
28+
var pricesData = new PricesData();
29+
30+
foreach (JsonProperty modelProperty in pricesElement.EnumerateObject())
31+
{
32+
var modelName = modelProperty.Name;
33+
if (modelProperty.Value.TryGetProperty("input", out JsonElement inputElement) &&
34+
modelProperty.Value.TryGetProperty("output", out JsonElement outputElement))
35+
{
36+
pricesData[modelName] = new()
37+
{
38+
Input = inputElement.GetDouble(),
39+
Output = outputElement.GetDouble()
40+
};
41+
}
42+
}
43+
44+
_configuration.Prices = pricesData;
45+
_logger.LogInformation("Language model prices data loaded from {PricesFile}", _configuration.PricesFile);
46+
}
47+
}
48+
catch (Exception ex)
49+
{
50+
_logger.LogError(ex, "An error has occurred while reading {PricesFile}:", _configuration.PricesFile);
51+
}
52+
}
53+
}

0 commit comments

Comments
 (0)