Skip to content

Commit b67dfe7

Browse files
dlucdluc
authored
Fix exception occurring when Azure OpenAI service is throttling requests (#876)
## Motivation and Context (Why the change? What's the scenario?) When retrying requests after throttling, the Azure OpenAI SDK is sending malformed requests, with multiple `Authorization` headers, which are rejected by the Azure OpenAI service with a `401 (Unauthorized)` error code, leading to an exception in the SDK. See - Azure/azure-sdk-for-net#46109 - microsoft/semantic-kernel#8929 - https://github.com/microsoft/kernel-memory/issues/855 ## High level description (Approach, Design) Inject a policy to fix malformed HTTP headers. Functional test included, to verify the fix.
1 parent 689cace commit b67dfe7

17 files changed

+216
-7
lines changed

‎.github/_typos.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ extend-exclude = [
1515
"encoder.json",
1616
"appsettings.development.json",
1717
"appsettings.Development.json",
18-
"AzureAISearchFilteringTest.cs"
18+
"AzureAISearchFilteringTest.cs",
19+
"KernelMemory.sln.DotSettings"
1920
]
2021

2122
[default.extend-words]

‎KernelMemory.sln‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureQueues", "extensions\A
187187
EndProject
188188
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureAIDocIntel", "extensions\AzureAIDocIntel\AzureAIDocIntel.csproj", "{CFE7C192-2561-40CC-8592-136293451EC1}"
189189
EndProject
190-
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureOpenAI", "extensions\AzureOpenAI\AzureOpenAI.csproj", "{93FA6DD6-D0B2-4751-8680-3F959E1F7AF2}"
190+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureOpenAI", "extensions\AzureOpenAI\AzureOpenAI\AzureOpenAI.csproj", "{93FA6DD6-D0B2-4751-8680-3F959E1F7AF2}"
191191
EndProject
192192
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureAISearch.TestApplication", "extensions\AzureAISearch\AzureAISearch.TestApplication\AzureAISearch.TestApplication.csproj", "{11445C36-1B94-4AFB-AC23-976C94924603}"
193193
EndProject
@@ -331,6 +331,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureAIContentSafety", "ext
331331
EndProject
332332
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KernelMemory", "extensions\KM\KernelMemory\KernelMemory.csproj", "{AB097B62-5A0B-4D74-9F8B-A41FE8241447}"
333333
EndProject
334+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureOpenAI.FunctionalTests", "extensions\AzureOpenAI\AzureOpenAI.FunctionalTests\AzureOpenAI.FunctionalTests.csproj", "{8E907766-4A7D-46E2-B5E3-EB2994B1AA54}"
335+
EndProject
334336
Global
335337
GlobalSection(SolutionConfigurationPlatforms) = preSolution
336338
Debug|Any CPU = Debug|Any CPU
@@ -613,6 +615,9 @@ Global
613615
{AB097B62-5A0B-4D74-9F8B-A41FE8241447}.Debug|Any CPU.Build.0 = Debug|Any CPU
614616
{AB097B62-5A0B-4D74-9F8B-A41FE8241447}.Release|Any CPU.ActiveCfg = Release|Any CPU
615617
{AB097B62-5A0B-4D74-9F8B-A41FE8241447}.Release|Any CPU.Build.0 = Release|Any CPU
618+
{8E907766-4A7D-46E2-B5E3-EB2994B1AA54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
619+
{8E907766-4A7D-46E2-B5E3-EB2994B1AA54}.Debug|Any CPU.Build.0 = Debug|Any CPU
620+
{8E907766-4A7D-46E2-B5E3-EB2994B1AA54}.Release|Any CPU.ActiveCfg = Release|Any CPU
616621
EndGlobalSection
617622
GlobalSection(SolutionProperties) = preSolution
618623
HideSolutionNode = FALSE
@@ -711,6 +716,7 @@ Global
711716
{82670921-FDCD-4672-84BD-4353F5AC24A0} = {3C17F42B-CFC8-4900-8CFB-88936311E919}
712717
{58E65B3F-EFF0-401A-AC76-A49835AE0220} = {155DA079-E267-49AF-973A-D1D44681970F}
713718
{AB097B62-5A0B-4D74-9F8B-A41FE8241447} = {155DA079-E267-49AF-973A-D1D44681970F}
719+
{8E907766-4A7D-46E2-B5E3-EB2994B1AA54} = {3C17F42B-CFC8-4900-8CFB-88936311E919}
714720
EndGlobalSection
715721
GlobalSection(ExtensibilityGlobals) = postSolution
716722
SolutionGuid = {CC136C62-115C-41D1-B414-F9473EFF6EA8}

‎applications/tests/Evaluation.Tests/Evaluation.FunctionalTests.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
</ItemGroup>
3434

3535
<ItemGroup>
36-
<ProjectReference Include="..\..\..\extensions\AzureOpenAI\AzureOpenAI.csproj" />
36+
<ProjectReference Include="..\..\..\extensions\AzureOpenAI\AzureOpenAI\AzureOpenAI.csproj" />
3737
<ProjectReference Include="..\..\evaluation\Evaluation.csproj" />
3838
</ItemGroup>
3939

‎extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/AzureOpenAI.FunctionalTests.csproj‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<Project Sdk="Microsoft.NET.Sdk.Web">
2+
3+
<PropertyGroup>
4+
<AssemblyName>Microsoft.AzureOpenAI.FunctionalTests</AssemblyName>
5+
<RootNamespace>Microsoft.AzureOpenAI.FunctionalTests</RootNamespace>
6+
<TargetFramework>net8.0</TargetFramework>
7+
<RollForward>LatestMajor</RollForward>
8+
<IsTestProject>true</IsTestProject>
9+
<ImplicitUsings>enable</ImplicitUsings>
10+
<Nullable>enable</Nullable>
11+
<IsPackable>false</IsPackable>
12+
<NoWarn>KMEXP01;</NoWarn>
13+
</PropertyGroup>
14+
15+
<ItemGroup>
16+
<ProjectReference Include="..\..\..\service\tests\TestHelpers\TestHelpers.csproj" />
17+
</ItemGroup>
18+
19+
<ItemGroup>
20+
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
21+
<PackageReference Include="Microsoft.NET.Test.Sdk" />
22+
<PackageReference Include="xunit" />
23+
<PackageReference Include="xunit.abstractions" />
24+
<PackageReference Include="Xunit.DependencyInjection" />
25+
<PackageReference Include="Xunit.DependencyInjection.Logging" />
26+
<PackageReference Include="xunit.runner.visualstudio">
27+
<PrivateAssets>all</PrivateAssets>
28+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
29+
</PackageReference>
30+
</ItemGroup>
31+
32+
</Project>

‎extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/Issue855Test.cs‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using Microsoft.KernelMemory.AI.AzureOpenAI;
4+
using Microsoft.KM.TestHelpers;
5+
using Xunit;
6+
using Xunit.Abstractions;
7+
8+
namespace Microsoft.AzureOpenAI.FunctionalTests;
9+
10+
/// <summary>
11+
/// References:
12+
/// - https://github.com/Azure/azure-sdk-for-net/issues/46109
13+
/// - https://github.com/microsoft/semantic-kernel/issues/8929
14+
/// - https://github.com/microsoft/kernel-memory/issues/855
15+
/// </summary>
16+
public class Issue855Test : BaseFunctionalTestCase
17+
{
18+
private readonly AzureOpenAITextEmbeddingGenerator _target;
19+
20+
public Issue855Test(IConfiguration cfg, ITestOutputHelper output) : base(cfg, output)
21+
{
22+
this._target = new AzureOpenAITextEmbeddingGenerator(this.AzureOpenAIEmbeddingConfiguration);
23+
}
24+
25+
[Fact(Skip = "Enable and run manually")]
26+
[Trait("Category", "Manual")]
27+
[Trait("Category", "BugFix")]
28+
public async Task ItDoesntWhenThrottling()
29+
{
30+
for (int i = 0; i < 50; i++)
31+
{
32+
Console.WriteLine($"## {i}");
33+
await this._target.GenerateEmbeddingBatchAsync(
34+
[RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr()]);
35+
}
36+
}
37+
38+
#pragma warning disable CA5394
39+
private static string RndStr()
40+
{
41+
var random = new Random();
42+
return new(Enumerable.Repeat(" ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ", 8000)
43+
.Select(s => s[random.Next(s.Length)]).ToArray());
44+
}
45+
}
46+
47+
#pragma warning disable IDE0055
48+
/* When the test fails: after pausing and trying to restart, an exception occurs.
49+
50+
Microsoft.SemanticKernel.HttpOperationException: Service request failed.
51+
52+
Microsoft.SemanticKernel.HttpOperationException
53+
Service request failed.
54+
Status: 401 (Unauthorized) <===== ******* Caused by https://github.com/Azure/azure-sdk-for-net/issues/46109
55+
56+
at Microsoft.SemanticKernel.Connectors.OpenAI.ClientCore.RunRequestAsync[T](Func`1 request)
57+
at Microsoft.SemanticKernel.Connectors.OpenAI.ClientCore.GetEmbeddingsAsync(String targetModel, IList`1 data, Kernel kernel, Nullable`1 dimensions, CancellationToken cancellationToken)
58+
at Microsoft.KernelMemory.AI.AzureOpenAI.AzureOpenAITextEmbeddingGenerator.GenerateEmbeddingBatchAsync(IEnumerable`1 textList, CancellationToken cancellationToken) in extensions/AzureOpenAI/AzureOpenAI/AzureOpenAITextEmbeddingGenerator.cs:line 132
59+
at Microsoft.AzureOpenAI.FunctionalTests.Issue855Test.ItDoesntFailWith401() in extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/Bug46109Test.cs:line 43
60+
at Xunit.DependencyInjection.DependencyInjectionTestInvoker.AsyncStack(Task task, Activity activity) in S:\GitHub\Xunit.DependencyInjection\src\Xunit.DependencyInjection\DependencyInjectionTestInvoker.cs:line 174
61+
62+
System.ClientModel.ClientResultException
63+
Service request failed.
64+
Status: 401 (Unauthorized)
65+
66+
at Azure.AI.OpenAI.ClientPipelineExtensions.ProcessMessageAsync(ClientPipeline pipeline, PipelineMessage message, RequestOptions options)
67+
at Azure.AI.OpenAI.Embeddings.AzureEmbeddingClient.GenerateEmbeddingsAsync(BinaryContent content, RequestOptions options)
68+
at OpenAI.Embeddings.EmbeddingClient.GenerateEmbeddingsAsync(IEnumerable`1 inputs, EmbeddingGenerationOptions options, CancellationToken cancellationToken)
69+
at Microsoft.SemanticKernel.Connectors.OpenAI.ClientCore.RunRequestAsync[T](Func`1 request)
70+
71+
72+
73+
warn: Microsoft.KernelMemory.AI.AzureOpenAI.AzureOpenAITextEmbeddingGenerator[0]
74+
Tokenizer not specified, will use GPT4oTokenizer. The token count might be incorrect, causing unexpected errors
75+
76+
## 0
77+
## 1
78+
## 2
79+
## 3
80+
...
81+
...
82+
warn: Microsoft.KernelMemory.AI.AzureOpenAI.Internals.ClientSequentialRetryPolicy[0]
83+
Header Retry-After found, value 21
84+
85+
warn: Microsoft.KernelMemory.AI.AzureOpenAI.Internals.ClientSequentialRetryPolicy[0]
86+
Delay extracted from HTTP response: 21000 msecs
87+
*/

‎extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/Startup.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
/* IMPORTANT: the Startup class must be at the root of the namespace and
4+
* the namespace must match exactly (required by Xunit.DependencyInjection) */
5+
6+
namespace Microsoft.AzureOpenAI.FunctionalTests;
7+
8+
public class Startup
9+
{
10+
public void ConfigureHost(IHostBuilder hostBuilder)
11+
{
12+
var config = new ConfigurationBuilder()
13+
.AddJsonFile("appsettings.json")
14+
.AddJsonFile("appsettings.development.json", optional: true)
15+
.AddJsonFile("appsettings.Development.json", optional: true)
16+
.AddUserSecrets<Startup>()
17+
.AddEnvironmentVariables()
18+
.Build();
19+
20+
hostBuilder.ConfigureHostConfiguration(builder => builder.AddConfiguration(config));
21+
}
22+
}

‎extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/appsettings.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"KernelMemory": {
3+
"Services": {
4+
"AzureOpenAIEmbedding": {
5+
"Auth": "AzureIdentity", // "ApiKey" or "AzureIdentity"
6+
"Endpoint": "https://<...>.openai.azure.com/",
7+
"APIKey": "",
8+
"Deployment": ""
9+
}
10+
}
11+
}
12+
}

extensions/AzureOpenAI/AzureOpenAI.csproj renamed to extensions/AzureOpenAI/AzureOpenAI/AzureOpenAI.csproj

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99
</PropertyGroup>
1010

1111
<ItemGroup>
12-
<ProjectReference Include="..\..\service\Abstractions\Abstractions.csproj" />
13-
<ProjectReference Include="..\OpenAI\OpenAI\OpenAI.csproj" />
12+
<ProjectReference Include="..\..\..\service\Abstractions\Abstractions.csproj" />
13+
<ProjectReference Include="..\..\OpenAI\OpenAI\OpenAI.csproj" />
1414
</ItemGroup>
1515

1616
<ItemGroup>
@@ -28,7 +28,7 @@
2828
</PropertyGroup>
2929

3030
<ItemGroup>
31-
<None Include="README.md" Link="README.md" Pack="true" PackagePath="." Visible="false" />
31+
<None Include="..\README.md" Link="README.md" Pack="true" PackagePath="." Visible="false" />
3232
</ItemGroup>
3333

3434
</Project>

extensions/AzureOpenAI/AzureOpenAIConfig.cs renamed to extensions/AzureOpenAI/AzureOpenAI/AzureOpenAIConfig.cs

File renamed without changes.

extensions/AzureOpenAI/AzureOpenAITextEmbeddingGenerator.cs renamed to extensions/AzureOpenAI/AzureOpenAI/AzureOpenAITextEmbeddingGenerator.cs

File renamed without changes.

0 commit comments

Comments
 (0)