Adds support for OAI Content Parts. Closes #1174 (#1175)

waldekmastykarz · web-flow · commit 4f636b1180bc · 2025-05-15T09:21:14.000+01:00
diff --git a/dev-proxy-abstractions/LanguageModel/ILanguageModelChatCompletionMessage.cs b/dev-proxy-abstractions/LanguageModel/ILanguageModelChatCompletionMessage.cs
@@ -6,6 +6,6 @@ namespace DevProxy.Abstractions.LanguageModel;
 
 public interface ILanguageModelChatCompletionMessage
 {
-    string Content { get; set; }
+    object Content { get; set; }
     string Role { get; set; }
 }
diff --git a/dev-proxy-abstractions/LanguageModel/OllamaModels.cs b/dev-proxy-abstractions/LanguageModel/OllamaModels.cs
@@ -89,7 +89,7 @@ public class OllamaLanguageModelChatCompletionResponse : OllamaResponse
     public OllamaLanguageModelChatCompletionMessage Message { get; set; } = new();
     public override string? Response
     {
-        get => Message.Content;
+        get => Message.Content.ToString();
         set
         {
             if (value is null)
@@ -118,7 +118,7 @@ public override OpenAIResponse ConvertToOpenAIResponse()
                 Index = 0,
                 Message = new()
                 {
-                    Content = Message.Content,
+                    Content = Message.Content.ToString() ?? string.Empty,
                     Role = Message.Role
                 }
             }],
@@ -152,7 +152,7 @@ public override OpenAIResponse ConvertToOpenAIResponse()
 
 public class OllamaLanguageModelChatCompletionMessage : ILanguageModelChatCompletionMessage
 {
-    public string Content { get; set; } = string.Empty;
+    public object Content { get; set; } = string.Empty;
     public string Role { get; set; } = string.Empty;
 
     public override bool Equals(object? obj)
diff --git a/dev-proxy-abstractions/LanguageModel/OpenAIContentPartJsonConverter.cs b/dev-proxy-abstractions/LanguageModel/OpenAIContentPartJsonConverter.cs
@@ -0,0 +1,75 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace DevProxy.Abstractions.LanguageModel;
+
+public class OpenAIContentPartJsonConverter : JsonConverter<object>
+{
+    public override bool CanConvert(Type typeToConvert)
+    {
+        return typeToConvert == typeof(object);
+    }
+
+    public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        if (reader.TokenType == JsonTokenType.String)
+        {
+            return reader.GetString();
+        }
+        else if (reader.TokenType == JsonTokenType.StartArray)
+        {
+            var contentParts = new List<object>();
+            while (reader.Read())
+            {
+                if (reader.TokenType == JsonTokenType.EndArray)
+                {
+                    break;
+                }
+
+                if (reader.TokenType == JsonTokenType.StartObject)
+                {
+                    using JsonDocument doc = JsonDocument.ParseValue(ref reader);
+
+                    var root = doc.RootElement;
+                    if (root.TryGetProperty("type", out var typeProp))
+                    {
+                        var contentType = typeProp.GetString() switch
+                        {
+                            "text" => typeof(OpenAITextContentPart),
+                            "image" => typeof(OpenAIImageContentPart),
+                            "audio" => typeof(OpenAIAudioContentPart),
+                            "file" => typeof(OpenAIFileContentPart),
+                            _ => null
+                        };
+                        if (contentType is not null)
+                        {
+                            var contentPart = JsonSerializer.Deserialize(doc.RootElement.GetRawText(), contentType, options);
+                            if (contentPart is not null)
+                            {
+                                contentParts.Add(contentPart);
+                            }
+                        }
+                    }
+                }
+            }
+            return contentParts.ToArray();
+        }
+        return null;
+    }
+
+    public override void Write(Utf8JsonWriter writer, object? value, JsonSerializerOptions options)
+    {
+        if (value is string str)
+        {
+            writer.WriteStringValue(str);
+        }
+        else
+        {
+            JsonSerializer.Serialize(writer, value, options);
+        }
+    }
+}
diff --git a/dev-proxy-abstractions/LanguageModel/OpenAILanguageModelClient.cs b/dev-proxy-abstractions/LanguageModel/OpenAILanguageModelClient.cs
@@ -2,9 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using Microsoft.Extensions.Logging;
 using System.Diagnostics;
 using System.Net.Http.Json;
-using Microsoft.Extensions.Logging;
 
 namespace DevProxy.Abstractions.LanguageModel;
 
@@ -170,7 +170,9 @@ private async Task<bool> IsEnabledInternalAsync()
                 Temperature = options?.Temperature
             };
 
-            var response = await _httpClient.PostAsJsonAsync(url, payload);
+            // var payloadString = JsonSerializer.Serialize(payload, ProxyUtils.JsonSerializerOptions);
+
+            var response = await _httpClient.PostAsJsonAsync(url, payload, ProxyUtils.JsonSerializerOptions);
             _logger.LogDebug("Response: {response}", response.StatusCode);
 
             if (!response.IsSuccessStatusCode)
diff --git a/dev-proxy-abstractions/LanguageModel/OpenAIModels.cs b/dev-proxy-abstractions/LanguageModel/OpenAIModels.cs
@@ -38,7 +38,7 @@ public class OpenAIError
     public string? Message { get; set; }
 }
 
-public abstract class OpenAIResponse: ILanguageModelCompletionResponse
+public abstract class OpenAIResponse : ILanguageModelCompletionResponse
 {
     public long Created { get; set; }
     public OpenAIError? Error { get; set; }
@@ -106,9 +106,63 @@ public class OpenAICompletionResponseChoice : OpenAIResponseChoice
     public string Text { get; set; } = string.Empty;
 }
 
-public class OpenAIChatCompletionMessage: ILanguageModelChatCompletionMessage
+#region content parts
+
+public abstract class OpenAIContentPart
 {
-    public string Content { get; set; } = string.Empty;
+    public string? Type { get; set; }
+}
+
+public class OpenAITextContentPart : OpenAIContentPart
+{
+    public string? Text { get; set; }
+}
+
+public class OpenAIImageContentPartUrl
+{
+    public string? Detail { get; set; } = "auto";
+    public string? Url { get; set; }
+}
+
+public class OpenAIImageContentPart : OpenAIContentPart
+{
+    [JsonPropertyName("image_url")]
+    public OpenAIImageContentPartUrl? Url { get; set; }
+}
+
+public class OpenAIAudioContentPartInputAudio
+{
+    public string? Data { get; set; }
+    public string? Format { get; set; }
+}
+
+public class OpenAIAudioContentPart : OpenAIContentPart
+{
+    [JsonPropertyName("input_audio")]
+    public OpenAIAudioContentPartInputAudio? InputAudio { get; set; }
+}
+
+public class OpenAIFileContentPartFile
+{
+    [JsonPropertyName("file_data")]
+    public string? Data { get; set; }
+    [JsonPropertyName("file_id")]
+    public string? Id { get; set; }
+    [JsonPropertyName("filename")]
+    public string? Name { get; set; }
+}
+
+public class OpenAIFileContentPart : OpenAIContentPart
+{
+    public OpenAIFileContentPartFile? File { get; set; }
+}
+
+#endregion
+
+public class OpenAIChatCompletionMessage : ILanguageModelChatCompletionMessage
+{
+    [JsonConverter(typeof(OpenAIContentPartJsonConverter))]
+    public object Content { get; set; } = string.Empty;
     public string Role { get; set; } = string.Empty;
 
     public override bool Equals(object? obj)
@@ -204,13 +258,13 @@ public class OpenAIFineTuneRequest : OpenAIRequest
 }
 
 public class OpenAIFineTuneResponse : OpenAIResponse
-{    
+{
     [JsonPropertyName("fine_tuned_model")]
-    public string? FineTunedModel { get; set; }   
-    public string Status { get; set; } = string.Empty;    
-    public string? Organization { get; set; }    
-    public long CreatedAt { get; set; }    
-    public long UpdatedAt { get; set; }    
+    public string? FineTunedModel { get; set; }
+    public string Status { get; set; } = string.Empty;
+    public string? Organization { get; set; }
+    public long CreatedAt { get; set; }
+    public long UpdatedAt { get; set; }
     [JsonPropertyName("training_file")]
     public string TrainingFile { get; set; } = string.Empty;
     [JsonPropertyName("validation_file")]
@@ -233,16 +287,16 @@ public class OpenAIImageRequest : OpenAIRequest
 }
 
 public class OpenAIImageResponse : OpenAIResponse
-{    
-    public OpenAIImageData[]? Data { get; set; }   
+{
+    public OpenAIImageData[]? Data { get; set; }
     public override string? Response => null; // Image responses don't have a text response
 }
 
 public class OpenAIImageData
 {
-    public string? Url { get; set; }    
+    public string? Url { get; set; }
     [JsonPropertyName("b64_json")]
-    public string? Base64Json { get; set; }    
+    public string? Base64Json { get; set; }
     [JsonPropertyName("revised_prompt")]
     public string? RevisedPrompt { get; set; }
 }
diff --git a/dev-proxy-plugins/Mocks/OpenAIMockResponsePlugin.cs b/dev-proxy-plugins/Mocks/OpenAIMockResponsePlugin.cs
@@ -34,6 +34,13 @@ public override async Task RegisterAsync()
 
     private async Task OnRequestAsync(object sender, ProxyRequestArgs e)
     {
+        if (UrlsToWatch is null ||
+            !e.HasRequestUrlMatch(UrlsToWatch))
+        {
+            Logger.LogRequest("URL not matched", MessageType.Skipped, new LoggingContext(e.Session));
+            return;
+        }
+
         var request = e.Session.HttpClient.Request;
         if (request.Method is null ||
             !request.Method.Equals("POST", StringComparison.OrdinalIgnoreCase) ||

Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,6 @@ namespace DevProxy.Abstractions.LanguageModel;`
`6`	`6`
`7`	`7`	`public interface ILanguageModelChatCompletionMessage`
`8`	`8`	`{`
`9`		`- string Content { get; set; }`
	`9`	`+ object Content { get; set; }`
`10`	`10`	`string Role { get; set; }`
`11`	`11`	`}`
Original file line number	Diff line number	Diff line change
`@@ -89,7 +89,7 @@ public class OllamaLanguageModelChatCompletionResponse : OllamaResponse`
`89`	`89`	`public OllamaLanguageModelChatCompletionMessage Message { get; set; } = new();`
`90`	`90`	`public override string? Response`
`91`	`91`	`{`
`92`		`- get => Message.Content;`
	`92`	`+ get => Message.Content.ToString();`
`93`	`93`	`set`
`94`	`94`	`{`
`95`	`95`	`if (value is null)`
`@@ -118,7 +118,7 @@ public override OpenAIResponse ConvertToOpenAIResponse()`
`118`	`118`	`Index = 0,`
`119`	`119`	`Message = new()`
`120`	`120`	`{`
`121`		`- Content = Message.Content,`
	`121`	`+ Content = Message.Content.ToString() ?? string.Empty,`
`122`	`122`	`Role = Message.Role`
`123`	`123`	`}`
`124`	`124`	`}],`
`@@ -152,7 +152,7 @@ public override OpenAIResponse ConvertToOpenAIResponse()`
`152`	`152`
`153`	`153`	`public class OllamaLanguageModelChatCompletionMessage : ILanguageModelChatCompletionMessage`
`154`	`154`	`{`
`155`		`- public string Content { get; set; } = string.Empty;`
	`155`	`+ public object Content { get; set; } = string.Empty;`
`156`	`156`	`public string Role { get; set; } = string.Empty;`
`157`	`157`
`158`	`158`	`public override bool Equals(object? obj)`