Onboard to test framework (#666)

* ignore failing tests

* move to nunit 4 patterns

* upgrade nunit

* first set of tests moved to ClientTestBase (auto sync/async)

* next set to ClientTestBase

* last few test files

* move to dev feed version

* add batch/chat tests

* update

* add chat tests

* add audio tests

* containers, assistants, responses

* models and moderations

* remaining tests

* be more flexible in test environment

* changes for pipeline

* feedback

* bump test proxy and test framework versions

* update recordings

* more recordings updates

* update more recordings

* merge conflicts and new recordings

Author: m-redding

.config/dotnet-tools.json

@@ -8,6 +8,13 @@
         "snippet-generator"
       ],
       "rollForward": false
+    },
+    "azure.sdk.tools.testproxy": {
+      "version": "1.0.0-dev.20250922.2",
+      "commands": [
+        "test-proxy"
+      ],
+      "rollForward": false
     }
   }
 }

tests/Assistants/Assistants.VectorStoresTests.cs

@@ -79,7 +79,7 @@ public void ResponseFormatEquality()
         Assert.That((AssistantResponseFormat)null != AssistantResponseFormat.CreateTextFormat());
         Assert.That(AssistantResponseFormat.CreateTextFormat() != null);
         Assert.That(AssistantResponseFormat.CreateTextFormat(), Is.Not.EqualTo(null));
-        Assert.That(null, Is.Not.EqualTo(AssistantResponseFormat.CreateTextFormat()));
+        Assert.That(AssistantResponseFormat.CreateTextFormat(), Is.Not.Null);
 
         AssistantResponseFormat jsonSchemaFormat = AssistantResponseFormat.CreateJsonSchemaFormat(
             name: "test_schema",

tests/Assistants/AssistantsSmokeTests.cs

@@ -1,18 +1,16 @@
 using System;
 using System.ClientModel;
 using System.Threading;
+using Microsoft.ClientModel.TestFramework;
 using NUnit.Framework;
 using OpenAI.Audio;
-using OpenAI.Tests.Utility;
 
 namespace OpenAI.Tests.Audio;
 
-[TestFixture(true)]
-[TestFixture(false)]
 [Parallelizable(ParallelScope.All)]
 [Category("Audio")]
 [Category("Smoke")]
-internal class GenerateSpeechMockTests : SyncAsyncTestBase
+internal class GenerateSpeechMockTests : ClientTestBase
 {
     private static readonly ApiKeyCredential s_fakeCredential = new ApiKeyCredential("key");
 
@@ -24,19 +22,11 @@ public GenerateSpeechMockTests(bool isAsync)
     [Test]
     public void GenerateSpeechRespectsTheCancellationToken()
     {
-        AudioClient client = new AudioClient("model", s_fakeCredential);
+        AudioClient client = CreateProxyFromClient(new AudioClient("model", s_fakeCredential));
         using CancellationTokenSource cancellationSource = new();
         cancellationSource.Cancel();
 
-        if (IsAsync)
-        {
-            Assert.That(async () => await client.GenerateSpeechAsync("text", GeneratedSpeechVoice.Echo, cancellationToken: cancellationSource.Token),
+        Assert.That(async () => await client.GenerateSpeechAsync("text", GeneratedSpeechVoice.Echo, cancellationToken: cancellationSource.Token),
                 Throws.InstanceOf<OperationCanceledException>());
-        }
-        else
-        {
-            Assert.That(() => client.GenerateSpeech("text", GeneratedSpeechVoice.Echo, cancellationToken: cancellationSource.Token),
-                Throws.InstanceOf<OperationCanceledException>());
-        }
     }
 }

tests/Assistants/AssistantsTests.cs

@@ -1,4 +1,5 @@
-using NUnit.Framework;
+using Microsoft.ClientModel.TestFramework;
+using NUnit.Framework;
 using OpenAI.Audio;
 using OpenAI.Tests.Utility;
 using System;
@@ -8,11 +9,8 @@
 
 namespace OpenAI.Tests.Audio;
 
-[TestFixture(true)]
-[TestFixture(false)]
-[Parallelizable(ParallelScope.All)]
 [Category("Audio")]
-public partial class GenerateSpeechTests : SyncAsyncTestBase
+public partial class GenerateSpeechTests : OpenAIRecordedTestBase
 {
     public GenerateSpeechTests(bool isAsync) : base(isAsync)
     {
@@ -21,14 +19,12 @@ public GenerateSpeechTests(bool isAsync) : base(isAsync)
     [Test]
     public async Task BasicTextToSpeechWorks()
     {
-        AudioClient client = GetTestClient<AudioClient>(TestScenario.Audio_TTS);
+        AudioClient client = GetProxiedOpenAIClient<AudioClient>(TestScenario.Audio_TTS);
 
-        BinaryData audio = IsAsync
-            ? await client.GenerateSpeechAsync("Hello, world! This is a test.", GeneratedSpeechVoice.Shimmer)
-            : client.GenerateSpeech("Hello, world! This is a test.", GeneratedSpeechVoice.Shimmer);
+        BinaryData audio = await client.GenerateSpeechAsync("Hello, world! This is a test.", GeneratedSpeechVoice.Shimmer);
 
         Assert.That(audio, Is.Not.Null);
-        ValidateGeneratedAudio(audio, "hello");
+        await ValidateGeneratedAudio(audio, "hello");
     }
 
     [Test]
@@ -41,7 +37,7 @@ public async Task BasicTextToSpeechWorks()
     [TestCase("pcm")]
     public async Task OutputFormatWorks(string responseFormat)
     {
-        AudioClient client = GetTestClient<AudioClient>(TestScenario.Audio_TTS);
+        AudioClient client = GetProxiedOpenAIClient<AudioClient>(TestScenario.Audio_TTS);
 
         SpeechGenerationOptions options = new();
 
@@ -59,9 +55,7 @@ public async Task OutputFormatWorks(string responseFormat)
             };
         }
 
-        BinaryData audio = IsAsync
-            ? await client.GenerateSpeechAsync("Hello, world!", GeneratedSpeechVoice.Alloy, options)
-            : client.GenerateSpeech("Hello, world!", GeneratedSpeechVoice.Alloy, options);
+        BinaryData audio = await client.GenerateSpeechAsync("Hello, world!", GeneratedSpeechVoice.Alloy, options);
 
         Assert.That(audio, Is.Not.Null);
 
@@ -82,10 +76,10 @@ public async Task OutputFormatWorks(string responseFormat)
         }
     }
 
-    private void ValidateGeneratedAudio(BinaryData audio, string expectedSubstring)
+    private async Task ValidateGeneratedAudio(BinaryData audio, string expectedSubstring)
     {
-        AudioClient client = GetTestClient<AudioClient>(TestScenario.Audio_Whisper);
-        AudioTranscription transcription = client.TranscribeAudio(audio.ToStream(), "hello_world.wav");
+        AudioClient client = GetProxiedOpenAIClient<AudioClient>(TestScenario.Audio_Whisper);
+        AudioTranscription transcription = await client.TranscribeAudioAsync(audio.ToStream(), "hello_world.wav");
 
         Assert.That(transcription.Text.ToLowerInvariant(), Contains.Substring(expectedSubstring));
     }

tests/Audio/GenerateSpeechMockTests.cs

@@ -4,18 +4,17 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.ClientModel.TestFramework;
+using Microsoft.ClientModel.TestFramework.Mocks;
 using NUnit.Framework;
 using OpenAI.Audio;
-using OpenAI.Tests.Utility;
 
 namespace OpenAI.Tests.Audio;
 
-[TestFixture(true)]
-[TestFixture(false)]
 [Parallelizable(ParallelScope.All)]
 [Category("Audio")]
 [Category("Smoke")]
-public partial class TranscriptionMockTests : SyncAsyncTestBase
+public partial class TranscriptionMockTests : ClientTestBase
 {
     private static readonly ApiKeyCredential s_fakeCredential = new ApiKeyCredential("key");
 
@@ -141,53 +140,43 @@ public async Task TranscribeAudioDeserializesSegment(AudioSourceKind audioSource
     [Test]
     public void TranscribeAudioFromStreamRespectsTheCancellationToken()
     {
-        AudioClient client = new AudioClient("model", s_fakeCredential);
+        AudioClient client = CreateProxyFromClient(new AudioClient("model", s_fakeCredential));
         using Stream stream = new MemoryStream();
         using CancellationTokenSource cancellationSource = new();
         cancellationSource.Cancel();
 
-        if (IsAsync)
-        {
-            Assert.That(async () => await client.TranscribeAudioAsync(stream, "filename", cancellationToken: cancellationSource.Token),
+        Assert.That(async () => await client.TranscribeAudioAsync(stream, "filename", cancellationToken: cancellationSource.Token),
                 Throws.InstanceOf<OperationCanceledException>());
-        }
-        else
-        {
-            Assert.That(() => client.TranscribeAudio(stream, "filename", cancellationToken: cancellationSource.Token),
-                Throws.InstanceOf<OperationCanceledException>());
-        }
     }
 
     private OpenAIClientOptions GetClientOptionsWithMockResponse(int status, string content)
     {
-        MockPipelineResponse response = new MockPipelineResponse(status);
-        response.SetContent(content);
+        MockPipelineResponse response = new MockPipelineResponse(status).WithContent(content);
 
         return new OpenAIClientOptions()
         {
-            Transport = new MockPipelineTransport(response)
+            Transport = new MockPipelineTransport(_ => response)
+            {
+                ExpectSyncPipeline = !IsAsync
+            }
         };
     }
 
     private async ValueTask<AudioTranscription> InvokeTranscribeAudioSyncOrAsync(OpenAIClientOptions clientOptions, AudioSourceKind audioSourceKind)
     {
-        AudioClient client = new AudioClient("model", s_fakeCredential, clientOptions);
+        AudioClient client = CreateProxyFromClient(new AudioClient("model", s_fakeCredential, clientOptions));
         string filename = "audio_french.wav";
         string path = Path.Combine("Assets", filename);
 
         if (audioSourceKind == AudioSourceKind.UsingStream)
         {
             using FileStream audio = File.OpenRead(path);
 
-            return IsAsync
-                ? await client.TranscribeAudioAsync(audio, filename)
-                : client.TranscribeAudio(audio, filename);
+            return await client.TranscribeAudioAsync(audio, filename);
         }
         else if (audioSourceKind == AudioSourceKind.UsingFilePath)
         {
-            return IsAsync
-                ? await client.TranscribeAudioAsync(path)
-                : client.TranscribeAudio(path);
+            return await client.TranscribeAudioAsync(path);
         }
 
         Assert.Fail("Invalid source kind.");

tests/Audio/GenerateSpeechTests.cs

@@ -1,4 +1,5 @@
-using NUnit.Framework;
+using Microsoft.ClientModel.TestFramework;
+using NUnit.Framework;
 using OpenAI.Audio;
 using OpenAI.Tests.Utility;
 using System;
@@ -8,16 +9,12 @@
 using System.IO;
 using System.Text;
 using System.Threading.Tasks;
-using System.Transactions;
 using static OpenAI.Tests.TestHelpers;
 
 namespace OpenAI.Tests.Audio;
 
-[TestFixture(true)]
-[TestFixture(false)]
-[Parallelizable(ParallelScope.All)]
 [Category("Audio")]
-public partial class TranscriptionTests : SyncAsyncTestBase
+public partial class TranscriptionTests : OpenAIRecordedTestBase
 {
     public TranscriptionTests(bool isAsync) : base(isAsync)
     {
@@ -34,7 +31,7 @@ public enum AudioSourceKind
     [TestCase(AudioSourceKind.UsingFilePath)]
     public async Task TranscriptionWorks(AudioSourceKind audioSourceKind)
     {
-        AudioClient client = GetTestClient<AudioClient>(TestScenario.Audio_Whisper);
+        AudioClient client = GetProxiedOpenAIClient<AudioClient>(TestScenario.Audio_Whisper);
         string filename = "audio_hello_world.mp3";
         string path = Path.Combine("Assets", filename);
         AudioTranscription transcription = null;
@@ -43,15 +40,11 @@ public async Task TranscriptionWorks(AudioSourceKind audioSourceKind)
         {
             using FileStream inputStream = File.OpenRead(path);
 
-            transcription = IsAsync
-                ? await client.TranscribeAudioAsync(inputStream, filename)
-                : client.TranscribeAudio(inputStream, filename);
+            transcription = await client.TranscribeAudioAsync(inputStream, filename);
         }
         else if (audioSourceKind == AudioSourceKind.UsingFilePath)
         {
-            transcription = IsAsync
-                ? await client.TranscribeAudioAsync(path)
-                : client.TranscribeAudio(path);
+            transcription = await client.TranscribeAudioAsync(path);
         }
 
         Assert.That(transcription, Is.Not.Null);
@@ -65,7 +58,7 @@ public async Task TranscriptionWorks(AudioSourceKind audioSourceKind)
     [TestCase(AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment)]
     public async Task TimestampsWork(AudioTimestampGranularities granularityFlags)
     {
-        AudioClient client = GetTestClient<AudioClient>(TestScenario.Audio_Whisper);
+        AudioClient client = GetProxiedOpenAIClient<AudioClient>(TestScenario.Audio_Whisper);
 
         using FileStream inputStream = File.OpenRead(Path.Combine("Assets", "audio_hello_world.mp3"));
 
@@ -76,9 +69,7 @@ public async Task TimestampsWork(AudioTimestampGranularities granularityFlags)
             TimestampGranularities = granularityFlags,
         };
 
-        ClientResult<AudioTranscription> transcriptionResult = IsAsync
-            ? await client.TranscribeAudioAsync(inputStream, "audio_hello_world.mp3", options)
-            : client.TranscribeAudio(inputStream, "audio_hello_world.mp3", options);
+        ClientResult<AudioTranscription> transcriptionResult = await client.TranscribeAudioAsync(inputStream, "audio_hello_world.mp3", options);
 
         PipelineResponse rawResponse = transcriptionResult.GetRawResponse();
         Assert.That(rawResponse.Content.ToString(), Is.Not.Null.And.Not.Empty);
@@ -140,7 +131,7 @@ public async Task TimestampsWork(AudioTimestampGranularities granularityFlags)
     [TestCase(null)]
     public async Task TranscriptionFormatsWork(string responseFormat)
     {
-        AudioClient client = GetTestClient<AudioClient>(TestScenario.Audio_Whisper);
+        AudioClient client = GetProxiedOpenAIClient<AudioClient>(TestScenario.Audio_Whisper);
         string path = Path.Combine("Assets", "audio_hello_world.mp3");
 
         AudioTranscriptionOptions options = new()
@@ -156,9 +147,7 @@ public async Task TranscriptionFormatsWork(string responseFormat)
             }
         };
 
-        AudioTranscription transcription = IsAsync
-            ? await client.TranscribeAudioAsync(path, options)
-            : client.TranscribeAudio(path, options);
+        AudioTranscription transcription = await client.TranscribeAudioAsync(path, options);
 
         Assert.That(transcription?.Text?.ToLowerInvariant(), Does.Contain("hello"));
 
@@ -180,7 +169,6 @@ public async Task TranscriptionFormatsWork(string responseFormat)
 
                 Assert.That(segment.Id, Is.EqualTo(i));
                 Assert.That(segment.EndTime, Is.GreaterThanOrEqualTo(segment.StartTime));
-                Assert.That(segment.TokenIds, Is.Not.Null);
                 Assert.That(segment.TokenIds.Length, Is.GreaterThan(0));
 
                 Assert.That(segment.AverageLogProbability, Is.LessThan(-0.001f).Or.GreaterThan(0.001f));
@@ -200,7 +188,7 @@ public async Task TranscriptionFormatsWork(string responseFormat)
     [Test]
     public async Task IncludesWork()
     {
-        AudioClient client = GetTestClient<AudioClient>(TestScenario.Audio_Gpt_4o_Mini_Transcribe);
+        AudioClient client = GetProxiedOpenAIClient<AudioClient>(TestScenario.Audio_Gpt_4o_Mini_Transcribe);
         string filename = "audio_hello_world.mp3";
         string path = Path.Combine("Assets", filename);
 
@@ -218,7 +206,7 @@ public async Task IncludesWork()
     [Test]
     public async Task StreamingIncludesWork()
     {
-        AudioClient client = GetTestClient<AudioClient>(TestScenario.Audio_Gpt_4o_Mini_Transcribe);
+        AudioClient client = GetProxiedOpenAIClient<AudioClient>(TestScenario.Audio_Gpt_4o_Mini_Transcribe);
         string filename = "audio_hello_world.mp3";
         string path = Path.Combine("Assets", filename);
 
@@ -252,7 +240,7 @@ in client.TranscribeAudioStreamingAsync(
     [Test]
     public async Task BadTranscriptionRequest()
     {
-        AudioClient client = GetTestClient<AudioClient>(TestScenario.Audio_Whisper);
+        AudioClient client = GetProxiedOpenAIClient<AudioClient>(TestScenario.Audio_Whisper);
 
         string path = Path.Combine("Assets", "audio_hello_world.mp3");
 
@@ -265,9 +253,7 @@ public async Task BadTranscriptionRequest()
 
         try
         {
-            _ = IsAsync
-                ? await client.TranscribeAudioAsync(path, options)
-                : client.TranscribeAudio(path, options);
+            await client.TranscribeAudioAsync(path, options);
         }
         catch (Exception ex)
         {
@@ -283,7 +269,7 @@ public async Task BadTranscriptionRequest()
     [TestCase(AudioSourceKind.UsingFilePath)]
     public async Task StreamingTranscriptionWorks(AudioSourceKind audioSourceKind)
     {
-        AudioClient client = GetTestClient<AudioClient>(TestScenario.Audio_Gpt_4o_Mini_Transcribe);
+        AudioClient client = GetProxiedOpenAIClient<AudioClient>(TestScenario.Audio_Gpt_4o_Mini_Transcribe);
         string filename = "audio_hello_world.mp3";
         string path = Path.Combine("Assets", filename);
 
@@ -328,7 +314,7 @@ public async Task StreamingTranscriptionWorks(AudioSourceKind audioSourceKind)
     [TestCase(AudioSourceKind.UsingFilePath)]
     public void StreamingTranscriptionThrowsForWhisperModel(AudioSourceKind audioSourceKind)
     {
-        AudioClient client = GetTestClient<AudioClient>(TestScenario.Audio_Whisper);
+        AudioClient client = GetProxiedOpenAIClient<AudioClient>(TestScenario.Audio_Whisper);
         string filename = "audio_hello_world.mp3";
         string path = Path.Combine("Assets", filename);