Skip to content

Commit 764e4ba

Browse files
alkampfergitalkampfergit
committed
Updated and compile
1 parent c2c66fc commit 764e4ba

File tree

7 files changed

+23
-145
lines changed

7 files changed

+23
-145
lines changed

‎Directory.Packages.props‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,21 @@
44
</PropertyGroup>
55
<ItemGroup>
66
<PackageVersion Include="Alkampfer.KernelMemory.ElasticSearch" Version="0.9.1" />
7-
<PackageVersion Include="Azure.AI.OpenAI" Version="1.0.0-beta.17" />
7+
<PackageVersion Include="Azure.AI.OpenAI" Version="2.0.0-beta.5" />
88
<PackageVersion Include="CommandDotNet.Spectre" Version="3.0.2" />
99
<PackageVersion Include="Microsoft.Extensions.Http" Version="8.0.0" />
10-
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="8.7.0" />
10+
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="8.9.1" />
1111
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
1212
<PackageVersion Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
13-
<PackageVersion Include="Microsoft.KernelMemory.Abstractions" Version="0.68.240716.1" />
14-
<PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.68.240716.1" />
15-
<PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24271.1" />
16-
<PackageVersion Include="Microsoft.SemanticKernel" Version="1.16.0" />
17-
<PackageVersion Include="Microsoft.SemanticKernel.PromptTemplates.Handlebars" Version="1.16.0" />
18-
<PackageVersion Include="Microsoft.SemanticKernel.Yaml" Version="1.16.0" />
19-
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />
20-
<PackageVersion Include="Microsoft.SemanticKernel.Core" Version="1.16.0" />
21-
<PackageVersion Include="Polly.Core" Version="8.4.1" />
13+
<PackageVersion Include="Microsoft.KernelMemory.Abstractions" Version="0.77.241004.1" />
14+
<PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.77.241004.1" />
15+
<PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24378.1" />
16+
<PackageVersion Include="Microsoft.SemanticKernel" Version="1.21.1" />
17+
<PackageVersion Include="Microsoft.SemanticKernel.PromptTemplates.Handlebars" Version="1.21.1" />
18+
<PackageVersion Include="Microsoft.SemanticKernel.Yaml" Version="1.21.1" />
19+
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.21.1" />
20+
<PackageVersion Include="Microsoft.SemanticKernel.Core" Version="1.21.1" />
21+
<PackageVersion Include="Polly.Core" Version="8.4.2" />
2222
<PackageVersion Include="TiktokenSharp" Version="1.1.4" />
2323
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
2424
</ItemGroup>
@@ -27,11 +27,11 @@
2727
<PackageVersion Include="coverlet.msbuild" Version="6.0.2" />
2828
<PackageVersion Include="fasterflect" Version="3.0.0" />
2929
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
30-
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
31-
<PackageVersion Include="xunit" Version="2.9.0" />
30+
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
31+
<PackageVersion Include="xunit" Version="2.9.2" />
3232
<PackageVersion Include="xunit.abstractions" Version="2.0.3" />
33-
<PackageVersion Include="Moq" Version="4.20.70" />
34-
<PackageVersion Include="Xunit.DependencyInjection" Version="9.3.0" />
33+
<PackageVersion Include="Moq" Version="4.20.72" />
34+
<PackageVersion Include="Xunit.DependencyInjection" Version="9.4.0" />
3535
<PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
3636
<PackageVersion Include="coverlet.collector" Version="6.0.2" />
3737
</ItemGroup>

‎src/KernelMemory.Extensions/Anthropic/AnthropicTextGeneration.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ internal class AnthropicTextGeneration : ITextGenerator
1818
/// We do not have cohere tokenizer directly in C# - in this first version we use gpt4 tokenizer
1919
/// and we know that this is a raw approximation but we only need to count.
2020
/// </summary>
21-
private static readonly Tokenizer _tokenizer = Tokenizer.CreateTiktokenForModel("gpt-4");
21+
private static readonly Tokenizer _tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");
2222

2323
public AnthropicTextGeneration(
2424
RawAnthropicHttpClient rawAnthropicHttpClient,
@@ -39,7 +39,7 @@ public int CountTokens(string text)
3939

4040
public IReadOnlyList<string> GetTokens(string text)
4141
{
42-
var tokens = _tokenizer.Encode(text, out var normalizedString);
42+
var tokens = _tokenizer.EncodeToTokens(text, out var normalizedString);
4343
return tokens.Select(t => t.Value).ToArray();
4444
}
4545

‎src/KernelMemory.Extensions/Cohere/CohereTokenizer.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ namespace KernelMemory.Extensions.Cohere;
1414
/// </summary>
1515
public class CohereTokenizer
1616
{
17-
public Dictionary<string, Tiktoken> Tokenizers { get; set; } = new();
17+
public Dictionary<string, TiktokenTokenizer> Tokenizers { get; set; } = new();
1818

1919
public CohereTokenizer(IHttpClientFactory httpClientFactory)
2020
{
@@ -31,7 +31,7 @@ public CohereTokenizer(IHttpClientFactory httpClientFactory)
3131
var extraData = File.ReadAllText(tokenizerExtraFile);
3232
var ed = JsonSerializer.Deserialize<ExtraTokenizerData>(extraData)!;
3333

34-
var tiktoken = new Tiktoken(tokenizerFile, null, specialTokens: ed.GetSpecialToken());
34+
var tiktoken = TiktokenTokenizer.Create(tokenizerFile, null, null, specialTokens: ed.GetSpecialToken());
3535
Tokenizers["command-r-plus"] = tiktoken;
3636
}
3737

‎src/KernelMemory.Extensions/Helper/AzureOpenaiEmbeddingGenerator.cs‎

Lines changed: 0 additions & 68 deletions
This file was deleted.

‎src/KernelMemory.Extensions/Helper/MicrosoftMlTiktokenTokenizer.cs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using Microsoft.KernelMemory.AI;
22
using Microsoft.ML.Tokenizers;
3-
using System;
43
using System.Collections.Generic;
54
using System.Linq;
65

@@ -18,7 +17,7 @@ public class MicrosoftMlTiktokenTokenizer : ITextTokenizer
1817

1918
public MicrosoftMlTiktokenTokenizer(string baseModelName)
2019
{
21-
_tikToken = Tiktoken.CreateTiktokenForModel(baseModelName);
20+
_tikToken = TiktokenTokenizer.CreateForModel(baseModelName);
2221
}
2322

2423
public int CountTokens(string text)
@@ -28,7 +27,7 @@ public int CountTokens(string text)
2827

2928
public IReadOnlyList<string> GetTokens(string text)
3029
{
31-
var tokens = _tikToken.Encode(text, out var normalizedString);
30+
var tokens = _tikToken.EncodeToTokens(text, out var normalizedString);
3231
return tokens.Select(t => t.Value).ToArray();
3332
}
3433
}

‎src/KernelMemory.Extensions/LocalLlm/ChatDtos.cs‎

Lines changed: 1 addition & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
using Azure.AI.OpenAI;
1+
using OpenAI.Chat;
22
using System;
33
using System.Collections.Generic;
44
using System.Text;
@@ -124,8 +124,6 @@ public class ToolCallFunction
124124

125125
public class Message
126126
{
127-
private readonly StreamingResponse<StreamingChatCompletionsUpdate> _streamingChatCompletions;
128-
129127
public Message(string role, string content)
130128
{
131129
Role = role;
@@ -137,37 +135,6 @@ public Message()
137135
{
138136
}
139137

140-
public Message(StreamingResponse<StreamingChatCompletionsUpdate> streamingChatCompletions)
141-
{
142-
_streamingChatCompletions = streamingChatCompletions;
143-
Content = "";
144-
Role = "assistant";
145-
Task.Run(ReadStreamResponse);
146-
}
147-
148-
private async Task ReadStreamResponse()
149-
{
150-
try
151-
{
152-
if (_streamingChatCompletions != null)
153-
{
154-
using (_streamingChatCompletions)
155-
{
156-
await foreach (var update in _streamingChatCompletions)
157-
{
158-
Content += update.ContentUpdate;
159-
OnContentChanged();
160-
}
161-
}
162-
}
163-
}
164-
catch (Exception e)
165-
{
166-
Console.WriteLine(e);
167-
throw;
168-
}
169-
}
170-
171138
public static Message CreateSystemMessage(string message)
172139
{
173140
return new Message("system", message);
@@ -205,26 +172,6 @@ private void OnContentChanged()
205172
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
206173
[JsonPropertyName("tool_call_id")]
207174
public string ToolCallId { get; set; }
208-
209-
public ChatRole GetChatRole()
210-
{
211-
if ("system".Equals(Role, StringComparison.OrdinalIgnoreCase))
212-
{
213-
return ChatRole.System;
214-
}
215-
else if ("assistant".Equals(Role, StringComparison.OrdinalIgnoreCase))
216-
{
217-
return ChatRole.Assistant;
218-
}
219-
else if ("user".Equals(Role, StringComparison.OrdinalIgnoreCase))
220-
{
221-
return ChatRole.User;
222-
}
223-
else
224-
{
225-
throw new System.Exception("Unknown role: " + Role);
226-
}
227-
}
228175
}
229176

230177

‎src/KernelMemory.Extensions/QueryPipeline/OpenaiRagQueryExecutor.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ public OpenaiRagQueryExecutor(
7171
{
7272
_kernel = kernel;
7373
_config = config ?? new OpenAIRagQueryExecutorConfiguration();
74-
_tokenizer = Tiktoken.CreateTiktokenForModel(_config.ModelName);
74+
_tokenizer = TiktokenTokenizer.CreateForModel(_config.ModelName);
7575
_log = log ?? DefaultLogger<StandardRagQueryExecutor>.Instance;
7676
}
7777

0 commit comments

Comments
 (0)