Skip to content

Commit daace7f

Browse files
waldekmastykarzgarrytrinder
waldekmastykarz
and
garrytrinder
authored
Adds OpenAIUsageDebuggingPlugin. Closes #1413 (#1427)
* Adds OpenAIUsageDebuggingPlugin. Closes #1413 * Make OpenAI usage logging more robust by safely parsing headers - Use DateTime.TryParse for the response Date header and fall back to DateTime.Now when missing/invalid. - Extract TryParseHeaderAsLong helper to centralize/parsing of numeric headers. - Replace inline parsing for x-ratelimit-remaining-tokens and x-ratelimit-remaining-requests with the new helper. - Add missing using for Titanium.Web.Proxy.Http. --------- Co-authored-by: Garry Trinder <[email protected]>
1 parent c301911 commit daace7f

File tree

4 files changed

+350
-150
lines changed

4 files changed

+350
-150
lines changed

‎DevProxy.Abstractions/LanguageModel/OpenAIModels.cs‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5+
using DevProxy.Abstractions.Utils;
6+
using Microsoft.Extensions.Logging;
7+
using System.Text.Json;
58
using System.Text.Json.Serialization;
69

710
namespace DevProxy.Abstractions.LanguageModel;
@@ -20,6 +23,95 @@ public class OpenAIRequest
2023
public double? Temperature { get; set; }
2124
[JsonPropertyName("top_p")]
2225
public double? TopP { get; set; }
26+
27+
public static bool TryGetOpenAIRequest(string content, ILogger logger, out OpenAIRequest? request)
28+
{
29+
logger.LogTrace("{Method} called", nameof(TryGetOpenAIRequest));
30+
31+
request = null;
32+
33+
if (string.IsNullOrEmpty(content))
34+
{
35+
logger.LogDebug("Request content is empty or null");
36+
return false;
37+
}
38+
39+
try
40+
{
41+
logger.LogDebug("Checking if the request is an OpenAI request...");
42+
43+
var rawRequest = JsonSerializer.Deserialize<JsonElement>(content, ProxyUtils.JsonSerializerOptions);
44+
45+
// Check for completion request (has "prompt", but not specific to image)
46+
if (rawRequest.TryGetProperty("prompt", out _) &&
47+
!rawRequest.TryGetProperty("size", out _) &&
48+
!rawRequest.TryGetProperty("n", out _))
49+
{
50+
logger.LogDebug("Request is a completion request");
51+
request = JsonSerializer.Deserialize<OpenAICompletionRequest>(content, ProxyUtils.JsonSerializerOptions);
52+
return true;
53+
}
54+
55+
// Chat completion request
56+
if (rawRequest.TryGetProperty("messages", out _))
57+
{
58+
logger.LogDebug("Request is a chat completion request");
59+
request = JsonSerializer.Deserialize<OpenAIChatCompletionRequest>(content, ProxyUtils.JsonSerializerOptions);
60+
return true;
61+
}
62+
63+
// Embedding request
64+
if (rawRequest.TryGetProperty("input", out _) &&
65+
rawRequest.TryGetProperty("model", out _) &&
66+
!rawRequest.TryGetProperty("voice", out _))
67+
{
68+
logger.LogDebug("Request is an embedding request");
69+
request = JsonSerializer.Deserialize<OpenAIEmbeddingRequest>(content, ProxyUtils.JsonSerializerOptions);
70+
return true;
71+
}
72+
73+
// Image generation request
74+
if (rawRequest.TryGetProperty("prompt", out _) &&
75+
(rawRequest.TryGetProperty("size", out _) || rawRequest.TryGetProperty("n", out _)))
76+
{
77+
logger.LogDebug("Request is an image generation request");
78+
request = JsonSerializer.Deserialize<OpenAIImageRequest>(content, ProxyUtils.JsonSerializerOptions);
79+
return true;
80+
}
81+
82+
// Audio transcription request
83+
if (rawRequest.TryGetProperty("file", out _))
84+
{
85+
logger.LogDebug("Request is an audio transcription request");
86+
request = JsonSerializer.Deserialize<OpenAIAudioRequest>(content, ProxyUtils.JsonSerializerOptions);
87+
return true;
88+
}
89+
90+
// Audio speech synthesis request
91+
if (rawRequest.TryGetProperty("input", out _) && rawRequest.TryGetProperty("voice", out _))
92+
{
93+
logger.LogDebug("Request is an audio speech synthesis request");
94+
request = JsonSerializer.Deserialize<OpenAIAudioSpeechRequest>(content, ProxyUtils.JsonSerializerOptions);
95+
return true;
96+
}
97+
98+
// Fine-tuning request
99+
if (rawRequest.TryGetProperty("training_file", out _))
100+
{
101+
logger.LogDebug("Request is a fine-tuning request");
102+
request = JsonSerializer.Deserialize<OpenAIFineTuneRequest>(content, ProxyUtils.JsonSerializerOptions);
103+
return true;
104+
}
105+
106+
logger.LogDebug("Request is not an OpenAI request.");
107+
return false;
108+
}
109+
catch (JsonException ex)
110+
{
111+
logger.LogDebug(ex, "Failed to deserialize OpenAI request.");
112+
return false;
113+
}
114+
}
23115
}
24116

25117
public class OpenAIResponse : ILanguageModelCompletionResponse
@@ -82,10 +174,18 @@ public class OpenAIResponseUsage
82174
public long CompletionTokens { get; set; }
83175
[JsonPropertyName("prompt_tokens")]
84176
public long PromptTokens { get; set; }
177+
[JsonPropertyName("prompt_tokens_details")]
178+
public PromptTokenDetails? PromptTokensDetails { get; set; }
85179
[JsonPropertyName("total_tokens")]
86180
public long TotalTokens { get; set; }
87181
}
88182

183+
public class PromptTokenDetails
184+
{
185+
[JsonPropertyName("cached_tokens")]
186+
public long CachedTokens { get; set; }
187+
}
188+
89189
public class OpenAIResponsePromptFilterResult
90190
{
91191
[JsonPropertyName("content_filter_results")]

‎DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs‎

Lines changed: 4 additions & 150 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
using DevProxy.Abstractions.Plugins;
88
using DevProxy.Abstractions.Proxy;
99
using DevProxy.Abstractions.Utils;
10+
using DevProxy.Plugins.Utils;
1011
using Microsoft.Extensions.Configuration;
1112
using Microsoft.Extensions.DependencyInjection;
1213
using Microsoft.Extensions.Logging;
@@ -19,7 +20,6 @@
1920
using System.Diagnostics;
2021
using System.Diagnostics.Metrics;
2122
using System.Text.Json;
22-
using Titanium.Web.Proxy.Http;
2323

2424
namespace DevProxy.Plugins.Inspection;
2525

@@ -108,7 +108,7 @@ public override Task BeforeRequestAsync(ProxyRequestArgs e, CancellationToken ca
108108
return Task.CompletedTask;
109109
}
110110

111-
if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest) || openAiRequest is null)
111+
if (!OpenAIRequest.TryGetOpenAIRequest(request.BodyString, Logger, out var openAiRequest) || openAiRequest is null)
112112
{
113113
Logger.LogRequest("Skipping non-OpenAI request", MessageType.Skipped, new(e.Session));
114114
return Task.CompletedTask;
@@ -323,9 +323,9 @@ private void ProcessSuccessResponse(Activity activity, ProxyResponseArgs e)
323323
}
324324

325325
var bodyString = response.BodyString;
326-
if (IsStreamingResponse(response))
326+
if (HttpUtils.IsStreamingResponse(response, Logger))
327327
{
328-
bodyString = GetBodyFromStreamingResponse(response);
328+
bodyString = HttpUtils.GetBodyFromStreamingResponse(response, Logger);
329329
}
330330

331331
AddResponseTypeSpecificTags(activity, openAiRequest, bodyString);
@@ -895,95 +895,6 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI
895895
Logger.LogTrace("RecordUsageMetrics() finished");
896896
}
897897

898-
private bool TryGetOpenAIRequest(string content, out OpenAIRequest? request)
899-
{
900-
Logger.LogTrace("TryGetOpenAIRequest() called");
901-
902-
request = null;
903-
904-
if (string.IsNullOrEmpty(content))
905-
{
906-
Logger.LogDebug("Request content is empty or null");
907-
return false;
908-
}
909-
910-
try
911-
{
912-
Logger.LogDebug("Checking if the request is an OpenAI request...");
913-
914-
var rawRequest = JsonSerializer.Deserialize<JsonElement>(content, ProxyUtils.JsonSerializerOptions);
915-
916-
// Check for completion request (has "prompt", but not specific to image)
917-
if (rawRequest.TryGetProperty("prompt", out _) &&
918-
!rawRequest.TryGetProperty("size", out _) &&
919-
!rawRequest.TryGetProperty("n", out _))
920-
{
921-
Logger.LogDebug("Request is a completion request");
922-
request = JsonSerializer.Deserialize<OpenAICompletionRequest>(content, ProxyUtils.JsonSerializerOptions);
923-
return true;
924-
}
925-
926-
// Chat completion request
927-
if (rawRequest.TryGetProperty("messages", out _))
928-
{
929-
Logger.LogDebug("Request is a chat completion request");
930-
request = JsonSerializer.Deserialize<OpenAIChatCompletionRequest>(content, ProxyUtils.JsonSerializerOptions);
931-
return true;
932-
}
933-
934-
// Embedding request
935-
if (rawRequest.TryGetProperty("input", out _) &&
936-
rawRequest.TryGetProperty("model", out _) &&
937-
!rawRequest.TryGetProperty("voice", out _))
938-
{
939-
Logger.LogDebug("Request is an embedding request");
940-
request = JsonSerializer.Deserialize<OpenAIEmbeddingRequest>(content, ProxyUtils.JsonSerializerOptions);
941-
return true;
942-
}
943-
944-
// Image generation request
945-
if (rawRequest.TryGetProperty("prompt", out _) &&
946-
(rawRequest.TryGetProperty("size", out _) || rawRequest.TryGetProperty("n", out _)))
947-
{
948-
Logger.LogDebug("Request is an image generation request");
949-
request = JsonSerializer.Deserialize<OpenAIImageRequest>(content, ProxyUtils.JsonSerializerOptions);
950-
return true;
951-
}
952-
953-
// Audio transcription request
954-
if (rawRequest.TryGetProperty("file", out _))
955-
{
956-
Logger.LogDebug("Request is an audio transcription request");
957-
request = JsonSerializer.Deserialize<OpenAIAudioRequest>(content, ProxyUtils.JsonSerializerOptions);
958-
return true;
959-
}
960-
961-
// Audio speech synthesis request
962-
if (rawRequest.TryGetProperty("input", out _) && rawRequest.TryGetProperty("voice", out _))
963-
{
964-
Logger.LogDebug("Request is an audio speech synthesis request");
965-
request = JsonSerializer.Deserialize<OpenAIAudioSpeechRequest>(content, ProxyUtils.JsonSerializerOptions);
966-
return true;
967-
}
968-
969-
// Fine-tuning request
970-
if (rawRequest.TryGetProperty("training_file", out _))
971-
{
972-
Logger.LogDebug("Request is a fine-tuning request");
973-
request = JsonSerializer.Deserialize<OpenAIFineTuneRequest>(content, ProxyUtils.JsonSerializerOptions);
974-
return true;
975-
}
976-
977-
Logger.LogDebug("Request is not an OpenAI request.");
978-
return false;
979-
}
980-
catch (JsonException ex)
981-
{
982-
Logger.LogDebug(ex, "Failed to deserialize OpenAI request.");
983-
return false;
984-
}
985-
}
986-
987898
private static string GetOperationName(OpenAIRequest request)
988899
{
989900
if (request == null)
@@ -1004,63 +915,6 @@ private static string GetOperationName(OpenAIRequest request)
1004915
};
1005916
}
1006917

1007-
private bool IsStreamingResponse(Response response)
1008-
{
1009-
Logger.LogTrace("{Method} called", nameof(IsStreamingResponse));
1010-
var contentType = response.Headers.FirstOrDefault(h => h.Name.Equals("content-type", StringComparison.OrdinalIgnoreCase))?.Value;
1011-
if (string.IsNullOrEmpty(contentType))
1012-
{
1013-
Logger.LogDebug("No content-type header found");
1014-
return false;
1015-
}
1016-
1017-
var isStreamingResponse = contentType.Contains("text/event-stream", StringComparison.OrdinalIgnoreCase);
1018-
Logger.LogDebug("IsStreamingResponse: {IsStreamingResponse}", isStreamingResponse);
1019-
1020-
Logger.LogTrace("{Method} finished", nameof(IsStreamingResponse));
1021-
return isStreamingResponse;
1022-
}
1023-
1024-
private string GetBodyFromStreamingResponse(Response response)
1025-
{
1026-
Logger.LogTrace("{Method} called", nameof(GetBodyFromStreamingResponse));
1027-
1028-
// default to the whole body
1029-
var bodyString = response.BodyString;
1030-
1031-
var chunks = bodyString.Split("\n\n", StringSplitOptions.RemoveEmptyEntries);
1032-
if (chunks.Length == 0)
1033-
{
1034-
Logger.LogDebug("No chunks found in the response body");
1035-
return bodyString;
1036-
}
1037-
1038-
// check if the last chunk is `data: [DONE]`
1039-
var lastChunk = chunks.Last().Trim();
1040-
if (lastChunk.Equals("data: [DONE]", StringComparison.OrdinalIgnoreCase))
1041-
{
1042-
// get next to last chunk
1043-
var chunk = chunks.Length > 1 ? chunks[^2].Trim() : string.Empty;
1044-
if (chunk.StartsWith("data: ", StringComparison.OrdinalIgnoreCase))
1045-
{
1046-
// remove the "data: " prefix
1047-
bodyString = chunk["data: ".Length..].Trim();
1048-
Logger.LogDebug("Last chunk starts with 'data: ', using the last chunk as the body: {BodyString}", bodyString);
1049-
}
1050-
else
1051-
{
1052-
Logger.LogDebug("Last chunk does not start with 'data: ', using the whole body");
1053-
}
1054-
}
1055-
else
1056-
{
1057-
Logger.LogDebug("Last chunk is not `data: [DONE]`, using the whole body");
1058-
}
1059-
1060-
Logger.LogTrace("{Method} finished", nameof(GetBodyFromStreamingResponse));
1061-
return bodyString;
1062-
}
1063-
1064918
public void Dispose()
1065919
{
1066920
_loader?.Dispose();

0 commit comments

Comments
 (0)