Skip to content

Commit bc0dac2

Browse files
alkampfergitalkampfergit
committed
Updated libraries, fixed tokenizers
1 parent a8af39b commit bc0dac2

File tree

8 files changed

+79
-18
lines changed

8 files changed

+79
-18
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ mono_crash.*
2020
# Virtual Environment
2121

2222
src/python/KernelMemory/
23+
src/python/kerkelmemoryextensions/
2324

2425
# Build results
2526
[Dd]ebug/

‎Directory.Packages.props‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,21 @@
33
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
44
</PropertyGroup>
55
<ItemGroup>
6-
<PackageVersion Include="Alkampfer.KernelMemory.ElasticSearch" Version="0.9.0-alpha0005" />
6+
<PackageVersion Include="Alkampfer.KernelMemory.ElasticSearch" Version="0.9.1" />
77
<PackageVersion Include="Azure.AI.OpenAI" Version="1.0.0-beta.17" />
88
<PackageVersion Include="CommandDotNet.Spectre" Version="3.0.2" />
99
<PackageVersion Include="Microsoft.Extensions.Http" Version="8.0.0" />
10-
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="8.6.0" />
10+
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="8.7.0" />
1111
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
1212
<PackageVersion Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
13-
<PackageVersion Include="Microsoft.KernelMemory.Abstractions" Version="0.65.240620.1" />
14-
<PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.65.240620.1" />
13+
<PackageVersion Include="Microsoft.KernelMemory.Abstractions" Version="0.68.240716.1" />
14+
<PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.68.240716.1" />
1515
<PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24271.1" />
16-
<PackageVersion Include="Microsoft.SemanticKernel" Version="1.15.1" />
17-
<PackageVersion Include="Microsoft.SemanticKernel.PromptTemplates.Handlebars" Version="1.15.1" />
18-
<PackageVersion Include="Microsoft.SemanticKernel.Yaml" Version="1.15.1" />
19-
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.15.1" />
20-
<PackageVersion Include="Microsoft.SemanticKernel.Core" Version="1.15.1" />
16+
<PackageVersion Include="Microsoft.SemanticKernel" Version="1.16.0" />
17+
<PackageVersion Include="Microsoft.SemanticKernel.PromptTemplates.Handlebars" Version="1.16.0" />
18+
<PackageVersion Include="Microsoft.SemanticKernel.Yaml" Version="1.16.0" />
19+
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />
20+
<PackageVersion Include="Microsoft.SemanticKernel.Core" Version="1.16.0" />
2121
<PackageVersion Include="Polly.Core" Version="8.4.1" />
2222
<PackageVersion Include="TiktokenSharp" Version="1.1.4" />
2323
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
@@ -28,11 +28,11 @@
2828
<PackageVersion Include="fasterflect" Version="3.0.0" />
2929
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
3030
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
31-
<PackageVersion Include="xunit" Version="2.8.1" />
31+
<PackageVersion Include="xunit" Version="2.9.0" />
3232
<PackageVersion Include="xunit.abstractions" Version="2.0.3" />
3333
<PackageVersion Include="Moq" Version="4.20.70" />
3434
<PackageVersion Include="Xunit.DependencyInjection" Version="9.3.0" />
35-
<PackageVersion Include="xunit.runner.visualstudio" Version="2.8.1" />
35+
<PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
3636
<PackageVersion Include="coverlet.collector" Version="6.0.2" />
3737
</ItemGroup>
3838
</Project>

‎src/KernelMemory.Extensions.ConsoleTest/Samples/CustomSearchPipelineBase.cs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using KernelMemory.Extensions;
33
using KernelMemory.Extensions.Cohere;
44
using KernelMemory.Extensions.ConsoleTest.Helper;
5+
using KernelMemory.Extensions.Helper;
56
using KernelMemory.Extensions.QueryPipeline;
67
using KernelMemory.Extensions.QueryPipeline.Diagnostic;
78
using Microsoft.Extensions.DependencyInjection;
@@ -249,12 +250,15 @@ private static IKernelMemoryBuilder CreateBasicKernelMemoryBuilder(
249250
Endpoint = Dotenv.Get("AZURE_ENDPOINT") ?? throw new ConfigurationException("AZURE_ENDPOINT missing from .env file"),
250251
APIType = AzureOpenAIConfig.APITypes.ChatCompletion,
251252
Auth = AzureOpenAIConfig.AuthTypes.APIKey,
252-
MaxTokenTotal = 4096
253+
MaxTokenTotal = 4096,
253254
};
254255

256+
//I need to override the tokenizer because kernel memory uses older version of Microsoft.Ml.Tokenizer
257+
var textGenerationTokenizer = new MicrosoftMlTiktokenTokenizer("gpt-4o");
258+
var embeddingTokenizer = new MicrosoftMlTiktokenTokenizer("text-embedding-ada-002");
255259
var kernelMemoryBuilder = new KernelMemoryBuilder(services)
256-
.WithAzureOpenAITextGeneration(chatConfig)
257-
.WithAzureOpenAITextEmbeddingGeneration(embeddingConfig);
260+
.WithAzureOpenAITextGeneration(chatConfig, textGenerationTokenizer)
261+
.WithAzureOpenAITextEmbeddingGeneration(embeddingConfig, embeddingTokenizer);
258262

259263
kernelMemoryBuilder
260264
.WithSimpleFileStorage(new SimpleFileStorageConfig()

‎src/KernelMemory.Extensions/Anthropic/AnthropicTextGeneration.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,12 @@ public int CountTokens(string text)
3737
return _tokenizer.CountTokens(text);
3838
}
3939

40+
public IReadOnlyList<string> GetTokens(string text)
41+
{
42+
var tokens = _tokenizer.Encode(text, out var normalizedString);
43+
return tokens.Select(t => t.Value).ToArray();
44+
}
45+
4046
/// <inheritdoc />
4147
public async IAsyncEnumerable<string> GenerateTextAsync(
4248
string prompt,

‎src/KernelMemory.Extensions/Helper/AzureOpenaiEmbeddingGenerator.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using KernelMemory.Extensions.Interfaces;
44
using Microsoft.KernelMemory;
55
using System;
6+
using System.Collections.Generic;
67
using System.Linq;
78
using System.Threading;
89
using System.Threading.Tasks;
@@ -43,6 +44,11 @@ public int CountTokens(string text)
4344
return _microsoftMlTiktokenTokenizer.CountTokens(text);
4445
}
4546

47+
public IReadOnlyList<string> GetTokens(string text)
48+
{
49+
return _microsoftMlTiktokenTokenizer.GetTokens(text);
50+
}
51+
4652
public async Task<Embedding> GenerateEmbeddingAsync(string text, CancellationToken cancellationToken = default)
4753
{
4854
var result = await GenerateEmbeddingsAsync([text], cancellationToken);

‎src/KernelMemory.Extensions/Helper/MicrosoftMlTiktokenTokenizer.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
using Microsoft.KernelMemory.AI;
22
using Microsoft.ML.Tokenizers;
33
using System;
4+
using System.Collections.Generic;
5+
using System.Linq;
46

57
namespace KernelMemory.Extensions.Helper
68
{
@@ -23,5 +25,11 @@ public int CountTokens(string text)
2325
{
2426
return _tikToken.CountTokens(text);
2527
}
28+
29+
public IReadOnlyList<string> GetTokens(string text)
30+
{
31+
var tokens = _tikToken.Encode(text, out var normalizedString);
32+
return tokens.Select(t => t.Value).ToArray();
33+
}
2634
}
2735
}

‎src/KernelMemory.Extensions/LocalLlm/ExternalEmbeddingGenerator.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,33 @@ public int CountTokens(string text)
8989
return tokenCountResult.count;
9090
}
9191

92+
public IReadOnlyList<string> GetTokens(string text)
93+
{
94+
var client = _httpClientFactory.CreateClient();
95+
//do a post request to the model to get the dense vector size
96+
var body = new
97+
{
98+
modelName = _embeddingGeneratorConfig.ModelName,
99+
text
100+
};
101+
var request = new HttpRequestMessage(HttpMethod.Post, $"{_embeddingGeneratorConfig.Address.TrimEnd('/')}/tokenize")
102+
{
103+
Content = new StringContent(JsonSerializer.Serialize(body), Encoding.UTF8, "application/json")
104+
};
105+
106+
var response = client.Send(request);
107+
108+
//TODO: Proper error handling
109+
if (!response.IsSuccessStatusCode)
110+
{
111+
throw new Exception("Failed to get dense vector size");
112+
}
113+
var responseStream = response.Content.ReadAsStream();
114+
var tokenCountResult = JsonSerializer.Deserialize<TokenizeResult>(responseStream);
115+
_log.LogDebug("[{countnum}]Tokenize of text of len {textlength} token number {tokencount}", Interlocked.Increment(ref _countTokenCallNum).ToString().PadLeft(8, ' '), text.Length, tokenCountResult!.tokens.Length);
116+
return tokenCountResult!.tokens.Select(t => t.value).ToArray();
117+
}
118+
92119
/// <inheritdoc />
93120
public async Task<Embedding> GenerateEmbeddingAsync(
94121
string text, CancellationToken cancellationToken = default)
@@ -128,6 +155,10 @@ public record EmbeddingModel(List<double[]> embeddings, string model);
128155

129156
private record EmbeddingInfo(string model, int dimension);
130157
private record TokenCountResult(int count);
158+
159+
private record TokenizeResult(Token[] tokens);
160+
161+
public record Token(int token, string value);
131162
}
132163

133164
public class ExternalEmbeddingGeneratorConfig

‎src/KernelMemory.Extensions/LocalLlm/LmStudioTextGeneration.cs‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
using KernelMemory.Extensions.LocalLlm;
1+
using KernelMemory.Extensions.Helper;
2+
using KernelMemory.Extensions.LocalLlm;
23
using Microsoft.KernelMemory.AI;
3-
using Microsoft.KernelMemory.AI.OpenAI;
44
using System;
55
using System.Collections.Generic;
66
using System.Net.Http;
@@ -15,7 +15,7 @@ public class LmStudioTextGeneration : ITextGenerator
1515
{
1616
private readonly IHttpClientFactory _httpClientFactory;
1717
private readonly Uri _baseUri;
18-
private readonly DefaultGPTTokenizer _textTokenizer;
18+
private readonly MicrosoftMlTiktokenTokenizer _textTokenizer;
1919

2020
public LmStudioTextGeneration(
2121
IHttpClientFactory httpClientFactory,
@@ -25,7 +25,7 @@ public LmStudioTextGeneration(
2525
_httpClientFactory = httpClientFactory;
2626
_baseUri = baseUri;
2727

28-
_textTokenizer = new DefaultGPTTokenizer();
28+
_textTokenizer = new MicrosoftMlTiktokenTokenizer("gpt-4");
2929
MaxTokenTotal = maxToken;
3030
}
3131

@@ -38,6 +38,11 @@ public int CountTokens(string text)
3838
return _textTokenizer.CountTokens(text);
3939
}
4040

41+
public IReadOnlyList<string> GetTokens(string text)
42+
{
43+
return _textTokenizer.GetTokens(text);
44+
}
45+
4146
/// <inheritdoc />
4247
public async IAsyncEnumerable<string> GenerateTextAsync(
4348
string prompt,

0 commit comments

Comments
 (0)