Skip to content

Commit 9219810

Browse files
author
Milder Hernandez
authored
Merge pull request #260 from milderhc/ms-learn
Add MS Learn code
2 parents 7d1d85d + 0c2ce39 commit 9219810

File tree

16 files changed

+691
-73
lines changed

16 files changed

+691
-73
lines changed

‎api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/Hotel.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public class Hotel {
4141
@VectorStoreRecordVector(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE)
4242
private final List<Float> indexedEuclidean;
4343

44-
@VectorStoreRecordData
44+
@VectorStoreRecordData(isFilterable = true)
4545
private final List<String> tags;
4646

4747
@VectorStoreRecordData

‎api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisVectorStoreTest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import com.microsoft.semantickernel.data.redis.RedisVectorStore;
77
import com.microsoft.semantickernel.data.redis.RedisVectorStoreOptions;
88
import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions;
9-
import com.microsoft.semantickernel.tests.data.jdbc.Hotel;
109
import com.redis.testcontainers.RedisContainer;
1110
import org.junit.jupiter.params.ParameterizedTest;
1211
import org.junit.jupiter.params.provider.EnumSource;

‎samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,10 @@
66
import com.azure.core.credential.AzureKeyCredential;
77
import com.azure.core.credential.KeyCredential;
88
import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService;
9-
import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue;
9+
import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults;
1010
import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection;
1111
import com.microsoft.semantickernel.data.VolatileVectorStore;
1212
import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions;
13-
import com.microsoft.semantickernel.data.VectorStoreTextSearch;
1413
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData;
1514
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey;
1615
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector;
@@ -43,7 +42,6 @@ static class GitHubFile {
4342
@VectorStoreRecordData
4443
private final String description;
4544
@VectorStoreRecordData
46-
@TextSearchResultValue
4745
private final String link;
4846
@VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE)
4947
private final List<Float> embedding;
@@ -125,24 +123,27 @@ public static void inMemoryStoreAndSearch(
125123
.then(storeData(collection, embeddingGeneration, sampleData()))
126124
.block();
127125

128-
// Build a vectorized search
129-
var vectorStoreTextSearch = VectorStoreTextSearch.<GitHubFile>builder()
130-
.withVectorizedSearch(collection)
131-
.withTextEmbeddingGenerationService(embeddingGeneration)
132-
.build();
133-
134126
// Search for results
135127
// Volatile store executes an exhaustive search, for approximate search use Azure AI Search, Redis or JDBC with PostgreSQL
136-
String query = "How to get started?";
137-
var results = vectorStoreTextSearch.searchAsync(query, null)
138-
.block();
128+
var results = search("How to get started", collection, embeddingGeneration).block();
139129

140130
if (results == null || results.getTotalCount() == 0) {
141131
System.out.println("No search results found.");
142132
return;
143133
}
134+
var searchResult = results.getResults().get(0);
135+
System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n",
136+
searchResult.getScore(), searchResult.getRecord().link,
137+
searchResult.getRecord().description);
138+
}
144139

145-
System.out.printf("Best result for '%s': %s%n", query, results.getResults().get(0));
140+
private static Mono<VectorSearchResults<GitHubFile>> search(
141+
String searchText,
142+
VectorStoreRecordCollection<String, GitHubFile> recordCollection,
143+
OpenAITextEmbeddingGenerationService embeddingGeneration) {
144+
// Generate embeddings for the search text and search for the closest records
145+
return embeddingGeneration.generateEmbeddingAsync(searchText)
146+
.flatMap(r -> recordCollection.searchAsync(r.getVector(), null));
146147
}
147148

148149
private static Mono<List<String>> storeData(

‎samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java‎

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,17 @@
55
import com.azure.ai.openai.OpenAIClientBuilder;
66
import com.azure.core.credential.AzureKeyCredential;
77
import com.azure.core.credential.KeyCredential;
8+
import com.azure.core.util.ClientOptions;
9+
import com.azure.core.util.MetricsOptions;
10+
import com.azure.core.util.TracingOptions;
811
import com.azure.search.documents.indexes.SearchIndexAsyncClient;
912
import com.azure.search.documents.indexes.SearchIndexClientBuilder;
10-
import com.fasterxml.jackson.annotation.JsonProperty;
1113
import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService;
1214
import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStore;
1315
import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStoreOptions;
1416
import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions;
15-
import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue;
17+
import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults;
1618
import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection;
17-
import com.microsoft.semantickernel.data.VectorStoreTextSearch;
1819
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData;
1920
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey;
2021
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector;
@@ -49,12 +50,11 @@ public class VectorStoreWithAzureAISearch {
4950
private static final int EMBEDDING_DIMENSIONS = 1536;
5051

5152
static class GitHubFile {
52-
@VectorStoreRecordKey()
53+
@VectorStoreRecordKey
5354
private final String id;
54-
@VectorStoreRecordData()
55+
@VectorStoreRecordData
5556
private final String description;
5657
@VectorStoreRecordData
57-
@TextSearchResultValue
5858
private final String link;
5959
@VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_SIMILARITY)
6060
private final List<Float> embedding;
@@ -64,10 +64,10 @@ public GitHubFile() {
6464
}
6565

6666
public GitHubFile(
67-
@JsonProperty("fileId") String id,
68-
@JsonProperty("description") String description,
69-
@JsonProperty("link") String link,
70-
@JsonProperty("embedding") List<Float> embedding) {
67+
String id,
68+
String description,
69+
String link,
70+
List<Float> embedding) {
7171
this.id = id;
7272
this.description = description;
7373
this.link = link;
@@ -108,6 +108,7 @@ public static void main(String[] args) {
108108
var searchClient = new SearchIndexClientBuilder()
109109
.endpoint(AZURE_AI_SEARCH_ENDPOINT)
110110
.credential(new AzureKeyCredential(AZURE_AISEARCH_KEY))
111+
.clientOptions(clientOptions())
111112
.buildAsyncClient();
112113

113114
storeAndSearch(searchClient, embeddingGeneration);
@@ -137,24 +138,27 @@ public static void storeAndSearch(
137138
.then(storeData(collection, embeddingGeneration, sampleData()))
138139
.block();
139140

140-
// Build a vectorized search
141-
var vectorStoreTextSearch = VectorStoreTextSearch.<GitHubFile>builder()
142-
.withVectorizedSearch(collection)
143-
.withTextEmbeddingGenerationService(embeddingGeneration)
144-
.build();
145-
146141
// Search for results
147142
// Might need to wait for the data to be indexed
148-
String query = "How to get started?";
149-
var results = vectorStoreTextSearch.searchAsync(query, null)
150-
.block();
143+
var results = search("How to get started", collection, embeddingGeneration).block();
151144

152145
if (results == null || results.getTotalCount() == 0) {
153146
System.out.println("No search results found.");
154147
return;
155148
}
149+
var searchResult = results.getResults().get(0);
150+
System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n",
151+
searchResult.getScore(), searchResult.getRecord().link,
152+
searchResult.getRecord().description);
153+
}
156154

157-
System.out.printf("Best result for '%s': %s%n", query, results.getResults().get(0));
155+
private static Mono<VectorSearchResults<GitHubFile>> search(
156+
String searchText,
157+
VectorStoreRecordCollection<String, GitHubFile> recordCollection,
158+
OpenAITextEmbeddingGenerationService embeddingGeneration) {
159+
// Generate embeddings for the search text and search for the closest records
160+
return embeddingGeneration.generateEmbeddingAsync(searchText)
161+
.flatMap(r -> recordCollection.searchAsync(r.getVector(), null));
158162
}
159163

160164
private static Mono<List<String>> storeData(
@@ -197,4 +201,11 @@ private static Map<String, String> sampleData() {
197201
"README: README associated with a sample chat summary react-based webapp" },
198202
}).collect(Collectors.toMap(element -> element[0], element -> element[1]));
199203
}
204+
205+
private static ClientOptions clientOptions() {
206+
return new ClientOptions()
207+
.setTracingOptions(new TracingOptions())
208+
.setMetricsOptions(new MetricsOptions())
209+
.setApplicationId("Semantic-Kernel");
210+
}
200211
}

‎samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java‎

Lines changed: 25 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,14 @@
99
import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore;
1010
import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions;
1111
import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions;
12-
import com.microsoft.semantickernel.data.jdbc.mysql.MySQLVectorStoreQueryProvider;
13-
import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue;
12+
import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider;
13+
import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults;
1414
import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection;
15-
import com.microsoft.semantickernel.data.VectorStoreTextSearch;
1615
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData;
1716
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey;
1817
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector;
1918
import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction;
20-
import com.mysql.cj.jdbc.MysqlDataSource;
19+
2120
import java.nio.charset.StandardCharsets;
2221
import java.sql.SQLException;
2322
import java.util.Arrays;
@@ -27,6 +26,7 @@
2726
import java.util.Map;
2827
import java.util.stream.Collectors;
2928

29+
import org.postgresql.ds.PGSimpleDataSource;
3030
import reactor.core.publisher.Flux;
3131
import reactor.core.publisher.Mono;
3232

@@ -42,12 +42,11 @@ public class VectorStoreWithJDBC {
4242
private static final int EMBEDDING_DIMENSIONS = 1536;
4343

4444
static class GitHubFile {
45-
@VectorStoreRecordKey()
45+
@VectorStoreRecordKey
4646
private final String id;
47-
@VectorStoreRecordData()
47+
@VectorStoreRecordData
4848
private final String description;
4949
@VectorStoreRecordData
50-
@TextSearchResultValue
5150
private final String link;
5251
@VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE)
5352
private final List<Float> embedding;
@@ -89,8 +88,8 @@ static String encodeId(String realId) {
8988
}
9089
}
9190

92-
// Run a MySQL server with:
93-
// docker run -d --name mysql-container -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=sk -p 3306:3306 mysql:latest
91+
// Run a PostgreSQL server with:
92+
// docker run -d --name pgvector-container -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=root -e POSTGRES_DB=sk -p 5432:5432 pgvector/pgvector:pg17
9493

9594
public static void main(String[] args) throws SQLException {
9695
System.out.println("==============================================================");
@@ -123,14 +122,14 @@ public static void main(String[] args) throws SQLException {
123122

124123
public static void storeAndSearch(OpenAITextEmbeddingGenerationService embeddingGeneration) {
125124
// Configure the data source
126-
var dataSource = new MysqlDataSource();
127-
dataSource.setUrl("jdbc:mysql://localhost:3306/sk");
125+
PGSimpleDataSource dataSource = new PGSimpleDataSource();
126+
dataSource.setUrl("jdbc:postgresql://localhost:5432/sk");
127+
dataSource.setUser("postgres");
128128
dataSource.setPassword("root");
129-
dataSource.setUser("root");
130129

131130
// Build a query provider
132131
// Other available query providers are PostgreSQLVectorStoreQueryProvider and SQLiteVectorStoreQueryProvider
133-
var queryProvider = MySQLVectorStoreQueryProvider.builder()
132+
var queryProvider = PostgreSQLVectorStoreQueryProvider.builder()
134133
.withDataSource(dataSource)
135134
.build();
136135

@@ -155,23 +154,26 @@ public static void storeAndSearch(OpenAITextEmbeddingGenerationService embedding
155154
.then(storeData(collection, embeddingGeneration, sampleData()))
156155
.block();
157156

158-
// Build a vectorized search
159-
var vectorStoreTextSearch = VectorStoreTextSearch.<GitHubFile>builder()
160-
.withVectorizedSearch(collection)
161-
.withTextEmbeddingGenerationService(embeddingGeneration)
162-
.build();
163-
164157
// Search for results
165-
String query = "How to get started?";
166-
var results = vectorStoreTextSearch.searchAsync(query, null)
167-
.block();
158+
var results = search("How to get started", collection, embeddingGeneration).block();
168159

169160
if (results == null || results.getTotalCount() == 0) {
170161
System.out.println("No search results found.");
171162
return;
172163
}
164+
var searchResult = results.getResults().get(0);
165+
System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n",
166+
searchResult.getScore(), searchResult.getRecord().link,
167+
searchResult.getRecord().description);
168+
}
173169

174-
System.out.printf("Best result for '%s': %s%n", query, results.getResults().get(0));
170+
private static Mono<VectorSearchResults<GitHubFile>> search(
171+
String searchText,
172+
VectorStoreRecordCollection<String, GitHubFile> recordCollection,
173+
OpenAITextEmbeddingGenerationService embeddingGeneration) {
174+
// Generate embeddings for the search text and search for the closest records
175+
return embeddingGeneration.generateEmbeddingAsync(searchText)
176+
.flatMap(r -> recordCollection.searchAsync(r.getVector(), null));
175177
}
176178

177179
private static Mono<List<String>> storeData(

‎samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,8 @@
1010
import com.microsoft.semantickernel.data.redis.RedisStorageType;
1111
import com.microsoft.semantickernel.data.redis.RedisVectorStore;
1212
import com.microsoft.semantickernel.data.redis.RedisVectorStoreOptions;
13-
import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue;
13+
import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults;
1414
import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection;
15-
import com.microsoft.semantickernel.data.VectorStoreTextSearch;
1615
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData;
1716
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey;
1817
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector;
@@ -42,12 +41,11 @@ public class VectorStoreWithRedis {
4241
private static final int EMBEDDING_DIMENSIONS = 1536;
4342

4443
public static class GitHubFile {
45-
@VectorStoreRecordKey()
44+
@VectorStoreRecordKey
4645
private final String id;
47-
@VectorStoreRecordData()
46+
@VectorStoreRecordData
4847
private final String description;
4948
@VectorStoreRecordData
50-
@TextSearchResultValue
5149
private final String link;
5250
@VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE)
5351
private final List<Float> embedding;
@@ -146,23 +144,27 @@ public static void storeAndSearch(
146144
.then(storeData(collection, embeddingGeneration, sampleData()))
147145
.block();
148146

149-
// Build a vectorized search
150-
var vectorStoreTextSearch = VectorStoreTextSearch.<GitHubFile>builder()
151-
.withVectorizedSearch(collection)
152-
.withTextEmbeddingGenerationService(embeddingGeneration)
153-
.build();
154-
155147
// Search for results
156-
String query = "How to get started?";
157-
var results = vectorStoreTextSearch.searchAsync(query, null)
158-
.block();
148+
// Might need to wait for the data to be indexed
149+
var results = search("How to get started", collection, embeddingGeneration).block();
159150

160151
if (results == null || results.getTotalCount() == 0) {
161152
System.out.println("No search results found.");
162153
return;
163154
}
155+
var searchResult = results.getResults().get(0);
156+
System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n",
157+
searchResult.getScore(), searchResult.getRecord().link,
158+
searchResult.getRecord().description);
159+
}
164160

165-
System.out.printf("Best result for '%s': %s%n", query, results.getResults().get(0));
161+
private static Mono<VectorSearchResults<GitHubFile>> search(
162+
String searchText,
163+
VectorStoreRecordCollection<String, GitHubFile> recordCollection,
164+
OpenAITextEmbeddingGenerationService embeddingGeneration) {
165+
// Generate embeddings for the search text and search for the closest records
166+
return embeddingGeneration.generateEmbeddingAsync(searchText)
167+
.flatMap(r -> recordCollection.searchAsync(r.getVector(), null));
166168
}
167169

168170
private static Mono<List<String>> storeData(

‎samples/semantickernel-learn-resources/pom.xml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,19 @@
2929
<artifactId>semantickernel-api</artifactId>
3030
</dependency>
3131

32+
<dependency>
33+
<groupId>com.microsoft.semantic-kernel</groupId>
34+
<artifactId>semantickernel-data-azureaisearch</artifactId>
35+
</dependency>
36+
<dependency>
37+
<groupId>com.microsoft.semantic-kernel</groupId>
38+
<artifactId>semantickernel-data-jdbc</artifactId>
39+
</dependency>
40+
<dependency>
41+
<groupId>com.microsoft.semantic-kernel</groupId>
42+
<artifactId>semantickernel-data-redis</artifactId>
43+
</dependency>
44+
3245
<dependency>
3346
<groupId>org.apache.logging.log4j</groupId>
3447
<artifactId>log4j-api</artifactId>

0 commit comments

Comments
 (0)