Skip to content

Commit ae7ee50

Browse files
author
Milder Hernandez Cagua
committed
Add sample and change Connection for DataSource
1 parent 4fe5a42 commit ae7ee50

File tree

17 files changed

+399
-140
lines changed

17 files changed

+399
-140
lines changed

‎aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818
/**
1919
* Provides OpenAi implementation of audio to text service.
2020
*/
21-
public class OpenAiAudioToTextService extends OpenAiService<OpenAIAsyncClient> implements AudioToTextService {
21+
public class OpenAiAudioToTextService extends OpenAiService<OpenAIAsyncClient>
22+
implements AudioToTextService {
2223

2324
private static final Logger LOGGER = LoggerFactory.getLogger(OpenAiAudioToTextService.class);
2425

‎aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@
1717
/**
1818
* Provides OpenAi implementation of text to audio service.
1919
*/
20-
public class OpenAiTextToAudioService extends OpenAiService<OpenAIAsyncClient> implements TextToAudioService {
20+
public class OpenAiTextToAudioService extends OpenAiService<OpenAIAsyncClient>
21+
implements TextToAudioService {
2122

2223
private static final Logger LOGGER = LoggerFactory.getLogger(OpenAiTextToAudioService.class);
2324

‎aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,8 @@
7979
/**
8080
* OpenAI chat completion service.
8181
*/
82-
public class OpenAIChatCompletion extends OpenAiService<OpenAIAsyncClient> implements ChatCompletionService {
82+
public class OpenAIChatCompletion extends OpenAiService<OpenAIAsyncClient>
83+
implements ChatCompletionService {
8384

8485
private static final Logger LOGGER = LoggerFactory.getLogger(OpenAIChatCompletion.class);
8586

@@ -1055,7 +1056,8 @@ static ChatRequestMessage getChatRequestMessage(
10551056
/**
10561057
* Builder for creating a new instance of {@link OpenAIChatCompletion}.
10571058
*/
1058-
public static class Builder extends OpenAiServiceBuilder<OpenAIAsyncClient, OpenAIChatCompletion, Builder> {
1059+
public static class Builder
1060+
extends OpenAiServiceBuilder<OpenAIAsyncClient, OpenAIChatCompletion, Builder> {
10591061

10601062
@Override
10611063
public OpenAIChatCompletion build() {

‎aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@
3030
/**
3131
* An OpenAI implementation of a {@link TextGenerationService}.
3232
*/
33-
public class OpenAITextGenerationService extends OpenAiService<OpenAIAsyncClient> implements TextGenerationService {
33+
public class OpenAITextGenerationService extends OpenAiService<OpenAIAsyncClient>
34+
implements TextGenerationService {
3435

3536
private static final Logger LOGGER = LoggerFactory.getLogger(OpenAITextGenerationService.class);
3637

‎api-test/integration-tests/pom.xml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,9 @@
6868
<version>3.44.1.0</version>
6969
</dependency>
7070
<dependency>
71-
<groupId>com.mysql</groupId>
72-
<artifactId>mysql-connector-j</artifactId>
73-
<version>8.2.0</version>
71+
<groupId>mysql</groupId>
72+
<artifactId>mysql-connector-java</artifactId>
73+
<version>8.0.33</version>
7474
<scope>test</scope>
7575
</dependency>
7676

‎api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,15 @@
55
import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider;
66
import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions;
77
import com.microsoft.semantickernel.tests.connectors.memory.Hotel;
8+
import com.mysql.cj.jdbc.MysqlDataSource;
89
import org.junit.jupiter.api.BeforeAll;
910
import org.junit.jupiter.api.Test;
1011
import org.testcontainers.containers.MySQLContainer;
1112
import org.testcontainers.junit.jupiter.Container;
1213
import org.testcontainers.junit.jupiter.Testcontainers;
1314

1415
import javax.annotation.Nonnull;
16+
import javax.sql.DataSource;
1517
import java.sql.Connection;
1618
import java.sql.DriverManager;
1719
import java.sql.SQLException;
@@ -29,20 +31,23 @@ public class JDBCVectorStoreRecordCollectionTest {
2931
private static final MySQLContainer<?> CONTAINER = new MySQLContainer<>("mysql:5.7.34");
3032
private static final String MYSQL_USER = "test";
3133
private static final String MYSQL_PASSWORD = "test";
32-
private static Connection connection;
34+
private static MysqlDataSource dataSource;
3335
@BeforeAll
34-
static void setup() throws SQLException {
35-
connection = DriverManager.getConnection(CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWORD);
36+
static void setup() {
37+
dataSource = new MysqlDataSource();
38+
dataSource.setUrl(CONTAINER.getJdbcUrl());
39+
dataSource.setUser(MYSQL_USER);
40+
dataSource.setPassword(MYSQL_PASSWORD);
3641
}
3742

3843
private JDBCVectorStoreRecordCollection<Hotel> buildRecordCollection(@Nonnull String collectionName) {
3944
JDBCVectorStoreRecordCollection<Hotel> recordCollection = new JDBCVectorStoreRecordCollection<>(
40-
connection,
45+
dataSource,
4146
collectionName,
4247
JDBCVectorStoreRecordCollectionOptions.<Hotel>builder()
4348
.withRecordClass(Hotel.class)
4449
.withQueryProvider(MySQLVectorStoreQueryProvider.builder()
45-
.withConnection(connection)
50+
.withDataSource(dataSource)
4651
.build())
4752
.build());
4853

‎api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions;
55
import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider;
66
import com.microsoft.semantickernel.tests.connectors.memory.Hotel;
7+
import com.mysql.cj.jdbc.MysqlDataSource;
78
import org.junit.jupiter.api.BeforeAll;
89
import org.junit.jupiter.api.Test;
910
import org.testcontainers.containers.MySQLContainer;
@@ -26,24 +27,29 @@ public class JDBCVectorStoreTest {
2627
private static final MySQLContainer<?> CONTAINER = new MySQLContainer<>("mysql:5.7.34");
2728
private static final String MYSQL_USER = "test";
2829
private static final String MYSQL_PASSWORD = "test";
29-
private static Connection connection;
30+
private static MysqlDataSource dataSource;
3031

3132
@BeforeAll
32-
static void setup() throws SQLException {
33-
connection = DriverManager.getConnection(CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWORD);
33+
static void setup() {
34+
dataSource = new MysqlDataSource();
35+
dataSource.setUrl(CONTAINER.getJdbcUrl());
36+
dataSource.setUser(MYSQL_USER);
37+
dataSource.setPassword(MYSQL_PASSWORD);
3438
}
3539

3640
@Test
3741
public void getCollectionNamesAsync() {
38-
JDBCVectorStoreOptions options = JDBCVectorStoreOptions.builder()
39-
.withQueryProvider(MySQLVectorStoreQueryProvider.builder()
40-
.withConnection(connection)
41-
.build())
42+
MySQLVectorStoreQueryProvider queryProvider = MySQLVectorStoreQueryProvider.builder()
43+
.withDataSource(dataSource)
4244
.build();
4345

4446
JDBCVectorStore vectorStore = JDBCVectorStore.builder()
45-
.withConnection(connection)
46-
.withOptions(options)
47+
.withDataSource(dataSource)
48+
.withOptions(
49+
JDBCVectorStoreOptions.builder()
50+
.withQueryProvider(queryProvider)
51+
.build()
52+
)
4753
.build();
4854

4955
vectorStore.getCollectionNamesAsync().block();

‎samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@
8888
<version>1.1.0</version>
8989
<scope>compile</scope>
9090
</dependency>
91+
92+
<dependency>
93+
<groupId>mysql</groupId>
94+
<artifactId>mysql-connector-java</artifactId>
95+
<version>8.0.33</version>
96+
</dependency>
9197
</dependencies>
9298

9399
<profiles>

‎samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
package com.microsoft.semantickernel.samples.syntaxexamples.memory;
2+
3+
import com.azure.ai.openai.OpenAIAsyncClient;
4+
import com.azure.ai.openai.OpenAIClientBuilder;
5+
import com.azure.core.credential.AzureKeyCredential;
6+
import com.azure.core.credential.KeyCredential;
7+
import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService;
8+
import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore;
9+
import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions;
10+
import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider;
11+
import com.microsoft.semantickernel.data.VectorStoreRecordCollection;
12+
import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute;
13+
import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute;
14+
import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute;
15+
import com.mysql.cj.jdbc.MysqlDataSource;
16+
import reactor.core.publisher.Flux;
17+
import reactor.core.publisher.Mono;
18+
19+
import javax.sql.DataSource;
20+
import java.nio.charset.StandardCharsets;
21+
import java.sql.Connection;
22+
import java.sql.DriverManager;
23+
import java.sql.SQLException;
24+
import java.util.Arrays;
25+
import java.util.Base64;
26+
import java.util.Collections;
27+
import java.util.List;
28+
import java.util.Map;
29+
import java.util.stream.Collectors;
30+
31+
public class JDBC_DataStorage {
32+
33+
private static final String CLIENT_KEY = System.getenv("CLIENT_KEY");
34+
private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY");
35+
36+
// Only required if AZURE_CLIENT_KEY is set
37+
private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT");
38+
private static final String MODEL_ID = System.getenv()
39+
.getOrDefault("EMBEDDING_MODEL_ID", "text-embedding-3-large");
40+
private static final int EMBEDDING_DIMENSIONS = 1536;
41+
42+
// Run a MySQL server with:
43+
// docker run -d --name mysql-container -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=sk -p 3306:3306 mysql:latest
44+
45+
static class GitHubFile {
46+
@VectorStoreRecordKeyAttribute()
47+
private final String id;
48+
@VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding")
49+
private final String description;
50+
@VectorStoreRecordDataAttribute
51+
private final String link;
52+
@VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw")
53+
private final List<Float> embedding;
54+
55+
public GitHubFile() {
56+
this(null, null, null, Collections.emptyList());
57+
}
58+
59+
public GitHubFile(
60+
String id,
61+
String description,
62+
String link,
63+
List<Float> embedding) {
64+
this.id = id;
65+
this.description = description;
66+
this.link = link;
67+
this.embedding = embedding;
68+
}
69+
70+
public String getId() {
71+
return id;
72+
}
73+
74+
public String getDescription() {
75+
return description;
76+
}
77+
78+
static String encodeId(String realId) {
79+
byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8));
80+
return new String(bytes, StandardCharsets.UTF_8);
81+
}
82+
}
83+
84+
public static void main(String[] args) throws SQLException {
85+
System.out.println("==============================================================");
86+
System.out.println("========== JDBC Vector Store Example ==============");
87+
System.out.println("==============================================================");
88+
89+
OpenAIAsyncClient client;
90+
91+
if (AZURE_CLIENT_KEY != null) {
92+
client = new OpenAIClientBuilder()
93+
.credential(new AzureKeyCredential(AZURE_CLIENT_KEY))
94+
.endpoint(CLIENT_ENDPOINT)
95+
.buildAsyncClient();
96+
97+
} else {
98+
client = new OpenAIClientBuilder()
99+
.credential(new KeyCredential(CLIENT_KEY))
100+
.buildAsyncClient();
101+
}
102+
103+
var embeddingGeneration = OpenAITextEmbeddingGenerationService.builder()
104+
.withOpenAIAsyncClient(client)
105+
.withModelId(MODEL_ID)
106+
.withDimensions(EMBEDDING_DIMENSIONS)
107+
.build();
108+
109+
var dataSource = new MysqlDataSource();
110+
dataSource.setUrl("jdbc:mysql://localhost:3306/sk");
111+
dataSource.setPassword("root");
112+
dataSource.setUser("root");
113+
114+
dataStorageWithMySQL(dataSource, embeddingGeneration);
115+
}
116+
117+
public static void dataStorageWithMySQL(
118+
DataSource dataSource,
119+
OpenAITextEmbeddingGenerationService embeddingGeneration) {
120+
121+
// Build a query provider
122+
var queryProvider = MySQLVectorStoreQueryProvider.builder()
123+
.withDataSource(dataSource)
124+
.build();
125+
126+
// Create a new vector store
127+
var jdbcVectorStore = JDBCVectorStore.builder()
128+
.withDataSource(dataSource)
129+
.withOptions(JDBCVectorStoreOptions.builder()
130+
.withQueryProvider(queryProvider)
131+
.build())
132+
.build();
133+
134+
String collectionName = "skgithubfiles";
135+
var collection = jdbcVectorStore.getCollection(collectionName, GitHubFile.class,
136+
null);
137+
138+
// Create collection if it does not exist and store data
139+
List<String> ids = collection
140+
.createCollectionIfNotExistsAsync()
141+
.then(storeData(collection, embeddingGeneration, sampleData()))
142+
.block();
143+
144+
List<GitHubFile> data = collection.getBatchAsync(ids, null).block();
145+
146+
data.forEach(gitHubFile -> System.out.println("Retrieved: " + gitHubFile.getDescription()));
147+
}
148+
149+
private static Mono<List<String>> storeData(
150+
VectorStoreRecordCollection<String, GitHubFile> recordStore,
151+
OpenAITextEmbeddingGenerationService embeddingGeneration,
152+
Map<String, String> data) {
153+
154+
return Flux.fromIterable(data.entrySet())
155+
.flatMap(entry -> {
156+
System.out.println("Save '" + entry.getKey() + "' to memory.");
157+
158+
return embeddingGeneration
159+
.generateEmbeddingsAsync(Collections.singletonList(entry.getValue()))
160+
.flatMap(embeddings -> {
161+
GitHubFile gitHubFile = new GitHubFile(
162+
GitHubFile.encodeId(entry.getKey()),
163+
entry.getValue(),
164+
entry.getKey(),
165+
embeddings.get(0).getVector());
166+
return recordStore.upsertAsync(gitHubFile, null);
167+
});
168+
})
169+
.collectList();
170+
}
171+
172+
private static Map<String, String> sampleData() {
173+
return Arrays.stream(new String[][] {
174+
{ "https://github.com/microsoft/semantic-kernel/blob/main/README.md",
175+
"README: Installation, getting started with Semantic Kernel, and how to contribute" },
176+
{ "https://github.com/microsoft/semantic-kernel/blob/main/samples/notebooks/dotnet/02-running-prompts-from-file.ipynb",
177+
"Jupyter notebook describing how to pass prompts from a file to a semantic skill or function" },
178+
{ "https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT",
179+
"Sample demonstrating how to create a chat skill interfacing with ChatGPT" },
180+
{ "https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs",
181+
"C# class that defines a volatile embedding store" },
182+
{ "https://github.com/microsoft/semantic-kernel/blob/main/samples/dotnet/KernelHttpServer/README.md",
183+
"README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4" },
184+
{ "https://github.com/microsoft/semantic-kernel/blob/main/samples/apps/chat-summary-webapp-react/README.md",
185+
"README: README associated with a sample chat summary react-based webapp" },
186+
}).collect(Collectors.toMap(element -> element[0], element -> element[1]));
187+
}
188+
}

0 commit comments

Comments
 (0)