Skip to content

Commit a10e4eb

Browse files
StephenHodgsonStephenHodgson
authored
com.openai.unity 8.8.8 (#436)
- Optimize image texture loading - Allow setting `Responses.TextContent.Type` to `OutputText` for `Role.Assistant` messages by @TypeDefinition - Fix free-after-use crash with audio refactor - Fixed wrapped server sent event error object - Fixed ability to create MCPApprovalResponse for mcp tool approvals - Fixed MCPToolCall.Error deserialization - Updated default models - com.utilities.rest -> 5.1.1 - com.utilities.audio -> 3.0.2
1 parent 316efef commit a10e4eb

25 files changed

+236
-115
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ obj/
4646
*.suo
4747
/*.sln
4848
*.sln
49+
/*.slnx
50+
*.slnx
4951
*.user
5052
*.unityproj
5153
*.ipch

‎OpenAI/.editorconfig‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ csharp_new_line_before_finally = true
2222
csharp_new_line_before_open_brace = all
2323

2424
# Modifier preferences
25-
dotnet_style_require_accessibility_modifiers = for_non_interface_members:error
25+
dotnet_style_require_accessibility_modifiers = error
2626

2727
# Code-block preferences
2828
csharp_prefer_braces = true:error
@@ -33,8 +33,8 @@ dotnet_style_predefined_type_for_locals_parameters_members = true
3333

3434
# Code Style
3535
csharp_style_var_when_type_is_apparent = true
36-
3736
dotnet_sort_system_directives_first = false
37+
dotnet_analyzer_diagnostic.category-Style.severity = none
3838

3939
#### Resharper/Rider Rules ####
4040
# https://www.jetbrains.com/help/resharper/EditorConfig_Properties.html

‎OpenAI/Packages/com.openai.unity/Runtime/Audio/AudioEndpoint.cs‎

Lines changed: 17 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -34,22 +34,21 @@ public async Task<Tuple<string, AudioClip>> CreateSpeechAsync(SpeechRequest requ
3434
[Obsolete("use GetSpeechAsync with Func<SpeechClip, Task> overload")]
3535
public async Task<Tuple<string, AudioClip>> CreateSpeechStreamAsync(SpeechRequest request, Action<AudioClip> partialClipCallback, CancellationToken cancellationToken = default)
3636
{
37-
using var result = await GetSpeechAsync(request, speechClip =>
37+
using var result = await GetSpeechAsync(request, async speechClip =>
3838
{
3939
partialClipCallback.Invoke(speechClip.AudioClip);
40+
await Task.CompletedTask;
4041
}, cancellationToken);
4142
return Tuple.Create(result.CachePath, result.AudioClip);
4243
}
4344

4445
[Obsolete("use GetSpeechAsync with Func<SpeechClip, Task> overload")]
4546
public async Task<SpeechClip> GetSpeechAsync(SpeechRequest request, Action<SpeechClip> partialClipCallback, CancellationToken cancellationToken = default)
46-
{
47-
return await GetSpeechAsync(request, partialClipCallback: clip =>
47+
=> await GetSpeechAsync(request, partialClipCallback: async clip =>
4848
{
4949
partialClipCallback?.Invoke(clip);
50-
return Task.CompletedTask;
50+
await Task.CompletedTask;
5151
}, cancellationToken);
52-
}
5352

5453
/// <summary>
5554
/// Generates audio from the input text.
@@ -61,6 +60,11 @@ public async Task<SpeechClip> GetSpeechAsync(SpeechRequest request, Action<Speec
6160
[Function("Generates audio from the input text.")]
6261
public async Task<SpeechClip> GetSpeechAsync(SpeechRequest request, Func<SpeechClip, Task> partialClipCallback = null, CancellationToken cancellationToken = default)
6362
{
63+
if (request == null)
64+
{
65+
throw new ArgumentNullException(nameof(request));
66+
}
67+
6468
if (partialClipCallback != null && request.ResponseFormat != SpeechResponseFormat.PCM)
6569
{
6670
Debug.LogWarning("Speech streaming only supported with PCM response format. Overriding to PCM...");
@@ -77,7 +81,7 @@ public async Task<SpeechClip> GetSpeechAsync(SpeechRequest request, Func<SpeechC
7781
var payload = JsonConvert.SerializeObject(request, OpenAIClient.JsonSerializationOptions);
7882
string clipName;
7983

80-
if (string.IsNullOrEmpty(request?.Voice))
84+
if (string.IsNullOrEmpty(request.Voice))
8185
{
8286
throw new ArgumentNullException(nameof(request.Voice));
8387
}
@@ -89,7 +93,7 @@ public async Task<SpeechClip> GetSpeechAsync(SpeechRequest request, Func<SpeechC
8993
clipName = $"{voice}-{DateTime.UtcNow:yyyyMMddThhmmssfffff}.{ext}";
9094
}
9195

92-
Rest.TryGetDownloadCacheItem(clipName, out var cachedPath);
96+
var cachePath = Rest.GetCacheItemPath(clipName);
9397

9498
switch (request.ResponseFormat)
9599
{
@@ -101,15 +105,7 @@ public async Task<SpeechClip> GetSpeechAsync(SpeechRequest request, Func<SpeechC
101105
if (partialClipCallback != null && partialResponse.Data.Length > 0)
102106
{
103107
var partialClip = new SpeechClip($"{clipName}_{++part}", null, partialResponse.Data);
104-
105-
try
106-
{
107-
await partialClipCallback(partialClip).ConfigureAwait(false);
108-
}
109-
finally
110-
{
111-
partialClip.Dispose();
112-
}
108+
await partialClipCallback(partialClip).ConfigureAwait(true);
113109
}
114110
}, 8192, new RestParameters(client.DefaultRequestHeaders, debug: EnableDebug), cancellationToken);
115111
pcmResponse.Validate(EnableDebug);
@@ -119,17 +115,17 @@ public async Task<SpeechClip> GetSpeechAsync(SpeechRequest request, Func<SpeechC
119115
throw new Exception("No audio data received!");
120116
}
121117

122-
await File.WriteAllBytesAsync(cachedPath, pcmResponse.Data, cancellationToken).ConfigureAwait(true);
123-
return new SpeechClip(clipName, cachedPath, pcmResponse.Data);
118+
await File.WriteAllBytesAsync(cachePath, pcmResponse.Data, cancellationToken).ConfigureAwait(true);
119+
return new SpeechClip(clipName, cachePath, pcmResponse.Data);
124120
}
125121
default:
126122
{
127123
var audioResponse = await Rest.PostAsync(GetUrl("/speech"), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken);
128124
audioResponse.Validate(EnableDebug);
129-
await File.WriteAllBytesAsync(cachedPath, audioResponse.Data, cancellationToken).ConfigureAwait(true);
125+
await File.WriteAllBytesAsync(cachePath, audioResponse.Data, cancellationToken).ConfigureAwait(true);
130126
var audioType = request.ResponseFormat == SpeechResponseFormat.MP3 ? AudioType.MPEG : AudioType.WAV;
131-
var finalClip = await Rest.DownloadAudioClipAsync(cachedPath, audioType, fileName: clipName, parameters: new RestParameters(debug: EnableDebug), cancellationToken: cancellationToken);
132-
return new SpeechClip(clipName, cachedPath, finalClip);
127+
var finalClip = await Rest.DownloadAudioClipAsync(cachePath, audioType, fileName: clipName, parameters: new RestParameters(debug: EnableDebug), cancellationToken: cancellationToken);
128+
return new SpeechClip(clipName, cachePath, finalClip);
133129
}
134130
}
135131
}

‎OpenAI/Packages/com.openai.unity/Runtime/Audio/SpeechClip.cs‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ namespace OpenAI.Audio
1111
[Preserve]
1212
public sealed class SpeechClip : IDisposable
1313
{
14+
[Preserve]
15+
private SpeechClip() { }
16+
1417
[Preserve]
1518
internal SpeechClip(string name, string cachePath, AudioClip audioClip)
1619
{
@@ -30,7 +33,8 @@ internal SpeechClip(string name, string cachePath, byte[] audioData, int sampleR
3033
SampleRate = sampleRate;
3134
}
3235

33-
~SpeechClip() => Dispose();
36+
[Preserve]
37+
~SpeechClip() => Dispose(false);
3438

3539
[Preserve]
3640
public string Name { get; }
@@ -112,13 +116,23 @@ public float Length
112116
[Preserve]
113117
public static implicit operator string(SpeechClip clip) => clip?.CachePath;
114118

119+
[Preserve]
120+
private void Dispose(bool disposing)
121+
{
122+
if (disposing)
123+
{
124+
audioSamples?.Dispose();
125+
audioSamples = null;
126+
audioData?.Dispose();
127+
audioData = null;
128+
}
129+
}
130+
115131
[Preserve]
116132
public void Dispose()
117133
{
118-
audioSamples?.Dispose();
119-
audioSamples = null;
120-
audioData?.Dispose();
121-
audioData = null;
134+
Dispose(true);
135+
GC.SuppressFinalize(this);
122136
}
123137
}
124138
}

‎OpenAI/Packages/com.openai.unity/Runtime/Chat/AudioOutput.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,9 @@ private void Dispose(bool disposing)
137137
if (disposing)
138138
{
139139
audioSamples?.Dispose();
140-
AudioData.Dispose();
140+
audioSamples = null;
141+
audioData?.Dispose();
142+
audioData = null;
141143
}
142144
}
143145

‎OpenAI/Packages/com.openai.unity/Runtime/Extensions/TextureExtensions.cs‎

Lines changed: 73 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,94 @@
11
// Licensed under the MIT License. See LICENSE in the project root for license information.
22

3-
using System;
4-
using System.IO;
53
using System.Threading;
64
using System.Threading.Tasks;
5+
using Unity.Collections;
76
using UnityEngine;
7+
using Utilities.Extensions;
8+
using OpenAI.Images;
9+
using Utilities.Async;
10+
using System;
811
using Utilities.WebRequestRest;
912

13+
#if !PLATFORM_WEBGL
14+
using System.IO;
15+
#endif
16+
1017
namespace OpenAI.Extensions
1118
{
12-
internal static class TextureExtensions
19+
public static class TextureExtensions
1320
{
14-
public static async Task<(Texture2D, string)> ConvertFromBase64Async(string b64, bool debug, CancellationToken cancellationToken)
21+
internal static async Task<(Texture2D, Uri)> ConvertFromBase64Async(string b64, bool debug, CancellationToken cancellationToken)
1522
{
16-
var imageData = Convert.FromBase64String(b64);
23+
using var imageData = NativeArrayExtensions.FromBase64String(b64, Allocator.Persistent);
1724
#if PLATFORM_WEBGL
1825
var texture = new Texture2D(2, 2);
26+
#if UNITY_6000_0_OR_NEWER
1927
texture.LoadImage(imageData);
20-
return await Task.FromResult((texture, string.Empty));
2128
#else
22-
if (!Rest.TryGetDownloadCacheItem(b64, out var localFilePath))
29+
texture.LoadImage(imageData.ToArray());
30+
#endif // UNITY_6000_0_OR_NEWER
31+
return await Task.FromResult((texture, null as Uri));
32+
#else
33+
if (!Rest.TryGetDownloadCacheItem(b64, out Uri localUri))
2334
{
24-
await File.WriteAllBytesAsync(localFilePath, imageData, cancellationToken).ConfigureAwait(true);
25-
localFilePath = $"file://{localFilePath}";
35+
await using var fs = new FileStream(localUri.LocalPath, FileMode.Create, FileAccess.Write);
36+
await fs.WriteAsync(imageData, cancellationToken: cancellationToken);
2637
}
2738

28-
var texture = await Rest.DownloadTextureAsync(localFilePath, parameters: new RestParameters(debug: debug), cancellationToken: cancellationToken);
29-
Rest.TryGetDownloadCacheItem(b64, out var cachedPath);
30-
return (texture, cachedPath);
31-
#endif
39+
var texture = await Rest.DownloadTextureAsync(localUri.LocalPath, parameters: new RestParameters(debug: debug), cancellationToken: cancellationToken);
40+
Rest.TryGetDownloadCacheItem(b64, out Uri cachedUri);
41+
return (texture, cachedUri);
42+
#endif // !PLATFORM_WEBGL
43+
}
44+
45+
/// <summary>
46+
/// Loads a Texture2D from an ImageResult, handling base64, cached path, or URL.
47+
/// </summary>
48+
/// <param name="imageResult">The <see cref="ImageResult"/> to load the texture for.</param>
49+
/// <param name="debug">Optional, debug flag.</param>
50+
/// <param name="cancellationToken">Optional, <see cref="CancellationToken"/>.</param>
51+
/// <returns>
52+
/// A tuple containing the converted <see cref="Texture2D"/> and the cached file path as a <see cref="Uri"/>.
53+
/// </returns>
54+
public static async Task<(Texture2D, Uri)> LoadTextureAsync(this ImageResult imageResult, bool debug = false, CancellationToken cancellationToken = default)
55+
{
56+
await Awaiters.UnityMainThread;
57+
58+
if (imageResult.Texture.IsNull())
59+
{
60+
if (!string.IsNullOrWhiteSpace(imageResult.B64_Json))
61+
{
62+
var (texture, cachedUri) = await ConvertFromBase64Async(imageResult.B64_Json, debug, cancellationToken);
63+
imageResult.Texture = texture;
64+
imageResult.CachedPathUri = cachedUri;
65+
}
66+
else
67+
{
68+
Texture2D texture;
69+
Uri cachedPath;
70+
71+
if (imageResult.CachedPathUri != null)
72+
{
73+
texture = await Rest.DownloadTextureAsync(imageResult.CachedPathUri, parameters: new RestParameters(debug: debug), cancellationToken: cancellationToken);
74+
cachedPath = imageResult.CachedPathUri;
75+
}
76+
else if (imageResult.Uri != null)
77+
{
78+
texture = await Rest.DownloadTextureAsync(imageResult.Uri, parameters: new RestParameters(debug: debug), cancellationToken: cancellationToken);
79+
cachedPath = Rest.TryGetDownloadCacheItem(imageResult.Uri, out var path) ? path : null;
80+
}
81+
else
82+
{
83+
throw new InvalidOperationException("ImageResult does not contain valid image data.");
84+
}
85+
86+
imageResult.Texture = texture;
87+
imageResult.CachedPathUri = cachedPath;
88+
}
89+
}
90+
91+
return (imageResult.Texture, imageResult.CachedPathUri);
3292
}
3393
}
3494
}

‎OpenAI/Packages/com.openai.unity/Runtime/Images/ImageResult.cs‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ internal ImageResult(
1818
[JsonProperty("revised_prompt")] string revisedPrompt)
1919
{
2020
Url = url;
21+
22+
if (!string.IsNullOrWhiteSpace(url))
23+
{
24+
Uri = new Uri(url);
25+
}
26+
2127
B64_Json = b64_json;
2228
RevisedPrompt = revisedPrompt;
2329
}
@@ -26,6 +32,10 @@ internal ImageResult(
2632
[JsonProperty("url", DefaultValueHandling = DefaultValueHandling.Ignore)]
2733
public string Url { get; private set; }
2834

35+
[Preserve]
36+
[JsonIgnore]
37+
public Uri Uri { get; }
38+
2939
[Preserve]
3040
[JsonProperty("b64_json", DefaultValueHandling = DefaultValueHandling.Ignore)]
3141
public string B64_Json { get; private set; }
@@ -56,7 +66,12 @@ internal ImageResult(
5666

5767
[Preserve]
5868
[JsonIgnore]
59-
public string CachedPath { get; internal set; }
69+
[Obsolete("use CachedPathUri")]
70+
public string CachedPath => CachedPathUri?.ToString();
71+
72+
[Preserve]
73+
[JsonIgnore]
74+
public Uri CachedPathUri { get; internal set; }
6075

6176
[Preserve]
6277
[JsonIgnore]
@@ -75,9 +90,9 @@ internal ImageResult(
7590
[Preserve]
7691
public override string ToString()
7792
{
78-
if (!string.IsNullOrWhiteSpace(CachedPath))
93+
if (CachedPathUri != null)
7994
{
80-
return CachedPath;
95+
return CachedPathUri.ToString();
8196
}
8297

8398
if (!string.IsNullOrWhiteSpace(B64_Json))

‎OpenAI/Packages/com.openai.unity/Runtime/Images/ImagesEndpoint.cs‎

Lines changed: 5 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
using System.Threading;
1010
using System.Threading.Tasks;
1111
using UnityEngine;
12-
using Utilities.Async;
1312
using Utilities.WebRequestRest;
1413

1514
namespace OpenAI.Images
@@ -180,33 +179,15 @@ private async Task<IReadOnlyList<ImageResult>> DeserializeResponseAsync(Response
180179
}
181180

182181
await Rest.ValidateCacheDirectoryAsync();
183-
var downloads = imagesResponse.Results.Select(DownloadAsync).ToList();
184182

185-
async Task DownloadAsync(ImageResult result)
186-
{
187-
await Awaiters.UnityMainThread;
188-
189-
if (string.IsNullOrWhiteSpace(result.Url))
190-
{
191-
var (texture, cachePath) = await TextureExtensions.ConvertFromBase64Async(result.B64_Json, EnableDebug, cancellationToken);
192-
result.Texture = texture;
193-
result.CachedPath = cachePath;
194-
}
195-
else
196-
{
197-
result.Texture = await Rest.DownloadTextureAsync(result.Url, parameters: new RestParameters(debug: EnableDebug), cancellationToken: cancellationToken);
198-
199-
if (Rest.TryGetDownloadCacheItem(result.Url, out var cachedPath))
200-
{
201-
result.CachedPath = cachedPath;
202-
}
203-
}
204-
}
183+
Task<(Texture2D, Uri)> DownloadAsync(ImageResult result)
184+
=> result.LoadTextureAsync(debug: EnableDebug, cancellationToken);
205185

206-
await Task.WhenAll(downloads).ConfigureAwait(true);
186+
await Task.WhenAll(imagesResponse.Results.Select(DownloadAsync).ToList()).ConfigureAwait(true);
207187

208-
foreach (var result in imagesResponse.Results)
188+
for (var i = 0; i < imagesResponse.Results.Count; i++)
209189
{
190+
var result = imagesResponse.Results[i];
210191
result.CreatedAt = DateTimeOffset.FromUnixTimeSeconds(imagesResponse.CreatedAtUnixSeconds).UtcDateTime;
211192
result.Background = imagesResponse.Background;
212193
result.OutputFormat = imagesResponse.OutputFormat;

0 commit comments

Comments
 (0)