Skip to content

Commit 69d852e

Browse files
Milder Hernandezjohnoliver
Milder Hernandez
and
johnoliver
authored
Rename memory package to data (#96)
* Rename memory to data Add RedisVectorStoreRecordCollection builder() method * add redis example * Fix name * Updates * Updates * Fix null away warnings * Fix null away warnings --------- Co-authored-by: John Oliver <[email protected]>
1 parent f47fe45 commit 69d852e

24 files changed

+409
-131
lines changed

‎api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreRecordCollectionTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package com.microsoft.semantickernel.tests.connectors.memory.redis;
22

3-
import com.microsoft.semantickernel.connectors.memory.redis.RedisVectorStoreRecordCollection;
4-
import com.microsoft.semantickernel.connectors.memory.redis.RedisVectorStoreRecordCollectionOptions;
3+
import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreRecordCollection;
4+
import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreRecordCollectionOptions;
55
import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField;
66
import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition;
77
import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField;

‎api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,16 @@
11
package com.microsoft.semantickernel.tests.connectors.memory.redis;
22

3-
import com.microsoft.semantickernel.connectors.memory.redis.RedisVectorStore;
4-
import com.microsoft.semantickernel.connectors.memory.redis.RedisVectorStoreOptions;
3+
import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore;
4+
import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions;
55
import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition;
66
import com.microsoft.semantickernel.tests.connectors.memory.Hotel;
77
import com.redis.testcontainers.RedisContainer;
88
import org.junit.jupiter.api.BeforeAll;
99
import org.junit.jupiter.api.Test;
1010
import org.testcontainers.junit.jupiter.Container;
1111
import org.testcontainers.junit.jupiter.Testcontainers;
12-
import reactor.core.publisher.Mono;
1312
import redis.clients.jedis.JedisPooled;
1413

15-
import java.util.ArrayList;
1614
import java.util.Arrays;
1715
import java.util.List;
1816

‎samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
import com.microsoft.semantickernel.samples.syntaxexamples.functions.Example59_OpenAIFunctionCalling;
1818
import com.microsoft.semantickernel.samples.syntaxexamples.functions.Example60_AdvancedMethodFunctions;
1919
import com.microsoft.semantickernel.samples.syntaxexamples.java.KernelFunctionYaml_Example;
20-
import com.microsoft.semantickernel.samples.syntaxexamples.memory.AzureAISearch;
20+
import com.microsoft.semantickernel.samples.syntaxexamples.memory.AzureAISearch_DataStorage;
2121
import com.microsoft.semantickernel.samples.syntaxexamples.plugins.Example10_DescribeAllPluginsAndFunctions;
2222
import com.microsoft.semantickernel.samples.syntaxexamples.plugins.Example13_ConversationSummaryPlugin;
2323
import com.microsoft.semantickernel.samples.syntaxexamples.template.Example06_TemplateLanguage;
@@ -38,7 +38,7 @@ public class RunAll {
3838

3939
public static void main(String[] args) {
4040
List<MainMethod> mains = Arrays.asList(
41-
AzureAISearch::main,
41+
AzureAISearch_DataStorage::main,
4242
Example01_NativeFunctions::main,
4343
Example03_Arguments::main,
4444
Example05_InlineFunctionDefinition::main,

samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch.java renamed to samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,9 @@
1111
import com.azure.search.documents.indexes.SearchIndexAsyncClient;
1212
import com.azure.search.documents.indexes.SearchIndexClientBuilder;
1313
import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService;
14-
import com.microsoft.semantickernel.connectors.memory.azureaisearch.AzureAISearchVectorStore;
15-
import com.microsoft.semantickernel.connectors.memory.azureaisearch.AzureAISearchVectorStoreOptions;
16-
import com.microsoft.semantickernel.connectors.memory.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions;
17-
import com.microsoft.semantickernel.connectors.memory.azureaisearch.AzureAISearchVectorStoreRecordCollection;
14+
import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore;
15+
import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreOptions;
16+
import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollection;
1817
import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute;
1918
import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute;
2019
import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute;
@@ -29,7 +28,7 @@
2928
import java.util.Map;
3029
import java.util.stream.Collectors;
3130

32-
public class AzureAISearch {
31+
public class AzureAISearch_DataStorage {
3332
private static final String CLIENT_KEY = System.getenv("CLIENT_KEY");
3433
private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY");
3534

@@ -107,10 +106,10 @@ public static void main(String[] args) {
107106
.clientOptions(clientOptions())
108107
.buildAsyncClient();
109108

110-
documentSearchWithAzureAISearch(searchClient, embeddingGeneration);
109+
dataStorageWithAzureAISearch(searchClient, embeddingGeneration);
111110
}
112111

113-
public static void documentSearchWithAzureAISearch(
112+
public static void dataStorageWithAzureAISearch(
114113
SearchIndexAsyncClient searchClient,
115114
OpenAITextEmbeddingGenerationService embeddingGeneration) {
116115

‎samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
package com.microsoft.semantickernel.samples.syntaxexamples.memory;
3+
4+
import com.azure.ai.openai.OpenAIAsyncClient;
5+
import com.azure.ai.openai.OpenAIClientBuilder;
6+
import com.azure.core.credential.AzureKeyCredential;
7+
import com.azure.core.credential.KeyCredential;
8+
import com.azure.core.util.ClientOptions;
9+
import com.azure.core.util.MetricsOptions;
10+
import com.azure.core.util.TracingOptions;
11+
import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService;
12+
import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore;
13+
import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions;
14+
import com.microsoft.semantickernel.data.VectorStoreRecordCollection;
15+
import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute;
16+
import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute;
17+
import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute;
18+
import java.nio.charset.StandardCharsets;
19+
import java.util.Arrays;
20+
import java.util.Base64;
21+
import java.util.Collections;
22+
import java.util.List;
23+
import java.util.Map;
24+
import java.util.stream.Collectors;
25+
import reactor.core.publisher.Flux;
26+
import reactor.core.publisher.Mono;
27+
import redis.clients.jedis.JedisPooled;
28+
29+
public class Redis_DataStorage {
30+
31+
private static final String CLIENT_KEY = System.getenv("CLIENT_KEY");
32+
private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY");
33+
34+
// Only required if AZURE_CLIENT_KEY is set
35+
private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT");
36+
37+
private static final String MODEL_ID = System.getenv()
38+
.getOrDefault("EMBEDDING_MODEL_ID", "text-embedding-3-large");
39+
private static final int EMBEDDING_DIMENSIONS = 1536;
40+
41+
// Can start a test server with:
42+
// docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest
43+
private static final String REDIS_URL = "redis://127.0.0.1:6379";
44+
45+
public static class GitHubFile {
46+
47+
@VectorStoreRecordKeyAttribute()
48+
private final String id;
49+
@VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding")
50+
private final String description;
51+
@VectorStoreRecordDataAttribute
52+
private final String link;
53+
@VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw")
54+
private final List<Float> embedding;
55+
56+
public GitHubFile() {
57+
this(null, null, null, Collections.emptyList());
58+
}
59+
60+
public GitHubFile(
61+
String id,
62+
String description,
63+
String link,
64+
List<Float> embedding) {
65+
this.id = id;
66+
this.description = description;
67+
this.link = link;
68+
this.embedding = embedding;
69+
}
70+
71+
public String getId() {
72+
return id;
73+
}
74+
public String getDescription() {
75+
return description;
76+
}
77+
78+
static String encodeId(String realId) {
79+
byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8));
80+
return new String(bytes, StandardCharsets.UTF_8);
81+
}
82+
}
83+
84+
public static void main(String[] args) {
85+
System.out.println("==============================================================");
86+
System.out.println("========== Redis Vector Store Example ==============");
87+
System.out.println("==============================================================");
88+
89+
OpenAIAsyncClient client;
90+
91+
if (AZURE_CLIENT_KEY != null) {
92+
client = new OpenAIClientBuilder()
93+
.credential(new AzureKeyCredential(AZURE_CLIENT_KEY))
94+
.endpoint(CLIENT_ENDPOINT)
95+
.buildAsyncClient();
96+
97+
} else {
98+
client = new OpenAIClientBuilder()
99+
.credential(new KeyCredential(CLIENT_KEY))
100+
.buildAsyncClient();
101+
}
102+
103+
var embeddingGeneration = OpenAITextEmbeddingGenerationService.builder()
104+
.withOpenAIAsyncClient(client)
105+
.withModelId(MODEL_ID)
106+
.withDimensions(EMBEDDING_DIMENSIONS)
107+
.build();
108+
109+
dataStorageWithRedis(embeddingGeneration);
110+
}
111+
112+
public static void dataStorageWithRedis(
113+
OpenAITextEmbeddingGenerationService embeddingGeneration) {
114+
115+
JedisPooled jedis = new JedisPooled(REDIS_URL);
116+
117+
RedisVectorStore<GitHubFile> vectorStore = RedisVectorStore.<GitHubFile>builder()
118+
.withClient(jedis)
119+
.withOptions(new RedisVectorStoreOptions<>(GitHubFile.class, null))
120+
.build();
121+
122+
String collectionName = "skgithubfiles";
123+
var collection = vectorStore.getCollection(collectionName, null);
124+
125+
// Create collection if it does not exist and store data
126+
List<String> ids = collection
127+
.createCollectionIfNotExistsAsync()
128+
.then(storeData(collection, embeddingGeneration, sampleData()))
129+
.block();
130+
131+
List<GitHubFile> data = collection.getBatchAsync(ids, null).block();
132+
133+
data.forEach(gitHubFile -> System.out.println("Retrieved: " + gitHubFile.getDescription()));
134+
135+
// TODO: Implement search functionality using Redis.
136+
137+
}
138+
139+
private static Mono<List<String>> storeData(
140+
VectorStoreRecordCollection<String, GitHubFile> recordStore,
141+
OpenAITextEmbeddingGenerationService embeddingGeneration,
142+
Map<String, String> data) {
143+
144+
return Flux.fromIterable(data.entrySet())
145+
.flatMap(entry -> {
146+
System.out.println("Save '" + entry.getKey() + "' to memory.");
147+
148+
return embeddingGeneration
149+
.generateEmbeddingsAsync(Collections.singletonList(entry.getValue()))
150+
.flatMap(embeddings -> {
151+
GitHubFile gitHubFile = new GitHubFile(
152+
GitHubFile.encodeId(entry.getKey()),
153+
entry.getValue(),
154+
entry.getKey(),
155+
embeddings.get(0).getVector());
156+
return recordStore.upsertAsync(gitHubFile, null);
157+
});
158+
})
159+
.collectList();
160+
}
161+
162+
private static Map<String, String> sampleData() {
163+
return Arrays.stream(new String[][] {
164+
{ "https://github.com/microsoft/semantic-kernel/blob/main/README.md",
165+
"README: Installation, getting started with Semantic Kernel, and how to contribute" },
166+
{ "https://github.com/microsoft/semantic-kernel/blob/main/samples/notebooks/dotnet/02-running-prompts-from-file.ipynb",
167+
"Jupyter notebook describing how to pass prompts from a file to a semantic skill or function" },
168+
{ "https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT",
169+
"Sample demonstrating how to create a chat skill interfacing with ChatGPT" },
170+
{ "https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs",
171+
"C# class that defines a volatile embedding store" },
172+
{ "https://github.com/microsoft/semantic-kernel/blob/main/samples/dotnet/KernelHttpServer/README.md",
173+
"README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4" },
174+
{ "https://github.com/microsoft/semantic-kernel/blob/main/samples/apps/chat-summary-webapp-react/README.md",
175+
"README: README associated with a sample chat summary react-based webapp" },
176+
}).collect(Collectors.toMap(element -> element[0], element -> element[1]));
177+
}
178+
179+
private static ClientOptions clientOptions() {
180+
return new ClientOptions()
181+
.setTracingOptions(new TracingOptions())
182+
.setMetricsOptions(new MetricsOptions())
183+
.setApplicationId("Semantic-Kernel");
184+
}
185+
}

semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/memory/azureaisearch/AzureAISearchVectorStore.java renamed to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
// Copyright (c) Microsoft. All rights reserved.
2-
package com.microsoft.semantickernel.connectors.memory.azureaisearch;
2+
package com.microsoft.semantickernel.connectors.data.azureaisearch;
33

44
import com.azure.search.documents.indexes.SearchIndexAsyncClient;
55
import com.azure.search.documents.indexes.models.SearchIndex;
66
import com.microsoft.semantickernel.data.VectorStore;
7-
import com.microsoft.semantickernel.data.VectorStoreRecordCollection;
87
import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition;
98
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
109
import reactor.core.publisher.Mono;

semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/memory/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java renamed to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// Copyright (c) Microsoft. All rights reserved.
2-
package com.microsoft.semantickernel.connectors.memory.azureaisearch;
2+
package com.microsoft.semantickernel.connectors.data.azureaisearch;
33

44
import com.azure.search.documents.indexes.models.ExhaustiveKnnAlgorithmConfiguration;
55
import com.azure.search.documents.indexes.models.ExhaustiveKnnParameters;
@@ -14,8 +14,11 @@
1414
import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField;
1515
import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField;
1616

17+
import javax.annotation.Nonnull;
18+
import javax.annotation.Nullable;
1719
import java.time.OffsetDateTime;
1820
import java.util.List;
21+
import java.util.Objects;
1922

2023
public class AzureAISearchVectorStoreCollectionCreateMapping {
2124

@@ -28,7 +31,11 @@ private static String getAlgorithmConfigName(VectorStoreRecordVectorField vector
2831
}
2932

3033
private static VectorSearchAlgorithmMetric getAlgorithmMetric(
31-
VectorStoreRecordVectorField vectorField) {
34+
@Nonnull VectorStoreRecordVectorField vectorField) {
35+
if (vectorField.getDistanceFunction() == null) {
36+
return VectorSearchAlgorithmMetric.COSINE;
37+
}
38+
3239
switch (vectorField.getDistanceFunction()) {
3340
case COSINE_SIMILARITY:
3441
return VectorSearchAlgorithmMetric.COSINE;
@@ -43,7 +50,12 @@ private static VectorSearchAlgorithmMetric getAlgorithmMetric(
4350
}
4451

4552
private static VectorSearchAlgorithmConfiguration getAlgorithmConfig(
46-
VectorStoreRecordVectorField vectorField) {
53+
@Nonnull VectorStoreRecordVectorField vectorField) {
54+
if (vectorField.getIndexKind() == null) {
55+
return new HnswAlgorithmConfiguration(getAlgorithmConfigName(vectorField))
56+
.setParameters(new HnswParameters().setMetric(getAlgorithmMetric(vectorField)));
57+
}
58+
4759
switch (vectorField.getIndexKind()) {
4860
case HNSW:
4961
return new HnswAlgorithmConfiguration(getAlgorithmConfigName(vectorField))
@@ -65,6 +77,11 @@ public static SearchField mapKeyField(VectorStoreRecordKeyField keyField) {
6577
}
6678

6779
public static SearchField mapDataField(VectorStoreRecordDataField dataField) {
80+
if (dataField.getFieldType() == null) {
81+
throw new IllegalArgumentException(
82+
"Field type is required: " + dataField.getName());
83+
}
84+
6885
return new SearchField(dataField.getName(),
6986
getSearchFieldDataType(dataField.getFieldType()))
7087
.setFilterable(dataField.isFilterable());

semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/memory/azureaisearch/AzureAISearchVectorStoreOptions.java renamed to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreOptions.java

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
// Copyright (c) Microsoft. All rights reserved.
2-
package com.microsoft.semantickernel.connectors.memory.azureaisearch;
2+
package com.microsoft.semantickernel.connectors.data.azureaisearch;
3+
4+
import javax.annotation.Nonnull;
5+
import javax.annotation.Nullable;
36

47
public class AzureAISearchVectorStoreOptions<Record> {
58
private final Class<Record> recordClass;
9+
@Nullable
610
private final AzureAISearchVectorStoreRecordCollectionFactory<Record> vectorStoreRecordCollectionFactory;
711

812
/**
@@ -11,8 +15,8 @@ public class AzureAISearchVectorStoreOptions<Record> {
1115
* @param recordClass The record class.
1216
* @param vectorStoreRecordCollectionFactory The vector store record collection factory.
1317
*/
14-
public AzureAISearchVectorStoreOptions(Class<Record> recordClass,
15-
AzureAISearchVectorStoreRecordCollectionFactory<Record> vectorStoreRecordCollectionFactory) {
18+
public AzureAISearchVectorStoreOptions(@Nonnull Class<Record> recordClass,
19+
@Nullable AzureAISearchVectorStoreRecordCollectionFactory<Record> vectorStoreRecordCollectionFactory) {
1620
this.recordClass = recordClass;
1721
this.vectorStoreRecordCollectionFactory = vectorStoreRecordCollectionFactory;
1822
}
@@ -41,6 +45,7 @@ public Class<Record> getRecordClass() {
4145
*
4246
* @return the vector store record collection factory
4347
*/
48+
@Nullable
4449
public AzureAISearchVectorStoreRecordCollectionFactory<Record> getVectorStoreRecordCollectionFactory() {
4550
return vectorStoreRecordCollectionFactory;
4651
}
@@ -51,7 +56,9 @@ public AzureAISearchVectorStoreRecordCollectionFactory<Record> getVectorStoreRec
5156
* @param <Record> the record type
5257
*/
5358
public static class Builder<Record> {
59+
@Nullable
5460
private Class<Record> recordClass;
61+
@Nullable
5562
private AzureAISearchVectorStoreRecordCollectionFactory<Record> vectorStoreRecordCollectionFactory;
5663

5764
/**
@@ -83,6 +90,10 @@ public Builder<Record> withVectorStoreRecordCollectionFactory(
8390
* @return The Azure AI Search vector store options.
8491
*/
8592
public AzureAISearchVectorStoreOptions<Record> build() {
93+
if (recordClass == null) {
94+
throw new IllegalArgumentException("recordClass is required");
95+
}
96+
8697
return new AzureAISearchVectorStoreOptions<>(recordClass,
8798
vectorStoreRecordCollectionFactory);
8899
}

0 commit comments

Comments
 (0)