Skip to content

Commit ba6680d

Browse files
odelpozo-denodoodelpozo-denodo
authored
Add support for creating SQL Server embedding stores without requiring the embeddings table to be recreated. (#445)
* Add support for creating SQL Server embedding stores without requiring the embeddings table to be recreated. * Add support for creating SQL Server embedding stores without requiring the embeddings table to be recreated.
1 parent 689d1a8 commit ba6680d

File tree

5 files changed

+145
-12
lines changed

5 files changed

+145
-12
lines changed

‎embedding-stores/langchain4j-community-sqlserver/README.md‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,34 @@ EmbeddingStore<TextSegment> embeddingStore = SQLServerEmbeddingStore.dataSourceB
5353
.dataSource(myDataSource)
5454
.embeddingTable(EmbeddingTable.builder()
5555
.name("my_embedding_table")
56-
.createOption(CreateOption.CREATE) // Use CreateOption.CREATE_OR_REPLACE to replace the existing table
57-
.dimension(384) // Must specify dimension
56+
.createOption(CreateOption.CREATE)
57+
.dimension(384)
58+
.build())
59+
.build();
60+
```
61+
62+
The previous option will fail if the table already exists. In that case, you can use the CREATE_IF_NOT_EXISTS option:
63+
64+
```java
65+
EmbeddingStore<TextSegment> embeddingStore = SQLServerEmbeddingStore.dataSourceBuilder()
66+
.dataSource(myDataSource)
67+
.embeddingTable(EmbeddingTable.builder()
68+
.name("my_embedding_table")
69+
.createOption(CreateOption.CREATE_IF_NOT_EXISTS)
70+
.dimension(384)
71+
.build())
72+
.build();
73+
```
74+
75+
Finally, If you want to recreate the table, you can use the CREATE_OR_REPLACE option:
76+
77+
```java
78+
EmbeddingStore<TextSegment> embeddingStore = SQLServerEmbeddingStore.dataSourceBuilder()
79+
.dataSource(myDataSource)
80+
.embeddingTable(EmbeddingTable.builder()
81+
.name("my_embedding_table")
82+
.createOption(CreateOption.CREATE_OR_REPLACE)
83+
.dimension(384)
5884
.build())
5985
.build();
6086
```
@@ -148,6 +174,8 @@ SQLServerEmbeddingStore embeddingStore =
148174
.build();
149175
```
150176

177+
- Indexes created with `Index.jsonIndexBuilder()` do not support the `CreateOption.CREATE_IF_NOT_EXISTS` option.
178+
151179
## Limitations
152180

153181
- Vector indexing performance depends on data size and distribution

‎embedding-stores/langchain4j-community-sqlserver/src/main/java/dev/langchain4j/store/embedding/sqlserver/CreateOption.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@ public enum CreateOption {
77

88
/** No attempt is made to create the schema object. */
99
CREATE_NONE,
10-
/** A new schema object is created. */
10+
/** A new schema object is created. If the object already exists, an error is thrown.*/
1111
CREATE,
12+
/** An existing schema object is created only if it does not already exist. */
13+
CREATE_IF_NOT_EXISTS,
1214
/** An existing schema object is dropped and replaced with a new one. */
1315
CREATE_OR_REPLACE
1416
}

‎embedding-stores/langchain4j-community-sqlserver/src/main/java/dev/langchain4j/store/embedding/sqlserver/EmbeddingTable.java‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,8 @@ void create(DataSource dataSource) throws SQLException {
162162
statement.addBatch(getDropTableStatement());
163163
}
164164

165-
statement.addBatch(getCreateTableStatement());
165+
statement.addBatch(getCreateTableStatement(createOption == CreateOption.CREATE_IF_NOT_EXISTS));
166+
166167
statement.executeBatch();
167168
}
168169
}
@@ -215,16 +216,29 @@ public String getQualifiedTableName() {
215216
}
216217
}
217218

218-
private String getCreateTableStatement() {
219-
return String.format(
219+
private String getCreateTableStatement(boolean ifNotExists) {
220+
221+
String createSql = String.format(
220222
"""
221-
CREATE TABLE %s (
222-
%s NVARCHAR(36) PRIMARY KEY,
223-
%s VECTOR(%d),
224-
%s NVARCHAR(MAX),
225-
%s JSON)
226-
""",
223+
CREATE TABLE %s (
224+
%s NVARCHAR(36) PRIMARY KEY,
225+
%s VECTOR(%d),
226+
%s NVARCHAR(MAX),
227+
%s JSON)
228+
""",
227229
getQualifiedTableName(), idColumn, embeddingColumn, dimension, textColumn, metadataColumn);
230+
if (ifNotExists) {
231+
return String.format(
232+
"""
233+
IF OBJECT_ID(N'%s', N'U') IS NULL
234+
BEGIN
235+
%s
236+
END;
237+
""",
238+
getQualifiedTableName(), createSql);
239+
} else {
240+
return createSql;
241+
}
228242
}
229243

230244
private String getDropTableStatement() {

‎embedding-stores/langchain4j-community-sqlserver/src/main/java/dev/langchain4j/store/embedding/sqlserver/JSONIndexBuilder.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ public JSONIndexBuilder key(String key, Class<?> keyType) {
7575
*/
7676
@Override
7777
public Index build() {
78+
if (CreateOption.CREATE_IF_NOT_EXISTS == createOption) {
79+
throw new IllegalStateException("Unsupported CreateOption.CREATE_IF_NOT_EXISTS.");
80+
}
7881
return new Index(this);
7982
}
8083

‎embedding-stores/langchain4j-community-sqlserver/src/test/java/dev/langchain4j/store/embedding/sqlserver/SQLServerEmbeddingStoreConfigIT.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import dev.langchain4j.data.embedding.Embedding;
88
import dev.langchain4j.data.segment.TextSegment;
99
import dev.langchain4j.store.embedding.EmbeddingStore;
10+
import dev.langchain4j.store.embedding.sqlserver.exception.SQLServerLangChain4jException;
1011
import org.junit.jupiter.api.Test;
1112

1213
class SQLServerEmbeddingStoreConfigIT {
@@ -29,6 +30,80 @@ void basic_config_test() {
2930
assertNotNull(embeddingStore);
3031
}
3132

33+
@Test
34+
void create_if_not_exists_config_test() {
35+
SQLServerDataSource dataSource = getSqlServerDataSource();
36+
SQLServerEmbeddingStore embeddingStore = SQLServerEmbeddingStore.dataSourceBuilder()
37+
.dataSource(dataSource)
38+
.embeddingTable(EmbeddingTable.builder()
39+
.createOption(CreateOption.CREATE_OR_REPLACE)
40+
.name("my_embedding_table")
41+
.dimension(4)
42+
.build())
43+
.build();
44+
assertNotNull(embeddingStore);
45+
46+
SQLServerEmbeddingStore.Builder embeddingStoreBuilder = SQLServerEmbeddingStore.dataSourceBuilder()
47+
.dataSource(dataSource)
48+
.embeddingTable(EmbeddingTable.builder()
49+
.createOption(CreateOption.CREATE)
50+
.name("my_embedding_table")
51+
.dimension(4)
52+
.build());
53+
assertThrows(SQLServerLangChain4jException.class, embeddingStoreBuilder::build);
54+
55+
SQLServerEmbeddingStore embeddingStore2 = SQLServerEmbeddingStore.dataSourceBuilder()
56+
.dataSource(dataSource)
57+
.embeddingTable(EmbeddingTable.builder()
58+
.createOption(CreateOption.CREATE_IF_NOT_EXISTS)
59+
.name("my_embedding_table")
60+
.dimension(4)
61+
.build())
62+
.build();
63+
64+
assertNotNull(embeddingStore2);
65+
}
66+
67+
@Test
68+
void create_if_not_exists_custom_catalog_config_test() {
69+
SQLServerDataSource dataSource = getSqlServerDataSource();
70+
SQLServerEmbeddingStore embeddingStore = SQLServerEmbeddingStore.dataSourceBuilder()
71+
.dataSource(dataSource)
72+
.embeddingTable(EmbeddingTable.builder()
73+
.createOption(CreateOption.CREATE_OR_REPLACE)
74+
.catalogName("master")
75+
.schemaName("dbo")
76+
.name("my_embedding_table")
77+
.dimension(4)
78+
.build())
79+
.build();
80+
assertNotNull(embeddingStore);
81+
82+
SQLServerEmbeddingStore.Builder embeddingStoreBuilder = SQLServerEmbeddingStore.dataSourceBuilder()
83+
.dataSource(dataSource)
84+
.embeddingTable(EmbeddingTable.builder()
85+
.createOption(CreateOption.CREATE)
86+
.catalogName("master")
87+
.schemaName("dbo")
88+
.name("my_embedding_table")
89+
.dimension(4)
90+
.build());
91+
assertThrows(SQLServerLangChain4jException.class, embeddingStoreBuilder::build);
92+
93+
SQLServerEmbeddingStore embeddingStore2 = SQLServerEmbeddingStore.dataSourceBuilder()
94+
.dataSource(dataSource)
95+
.embeddingTable(EmbeddingTable.builder()
96+
.createOption(CreateOption.CREATE_IF_NOT_EXISTS)
97+
.catalogName("master")
98+
.schemaName("dbo")
99+
.name("my_embedding_table")
100+
.dimension(4)
101+
.build())
102+
.build();
103+
104+
assertNotNull(embeddingStore2);
105+
}
106+
32107
@Test
33108
void should_create_table_with_json_indexes() {
34109
SQLServerDataSource dataSource = getSqlServerDataSource();
@@ -55,6 +130,17 @@ void should_create_table_with_json_indexes() {
55130
assertNotNull(embeddingStore);
56131
}
57132

133+
@Test
134+
void index_with_create_if_not_exists_may_fail() {
135+
136+
JSONIndexBuilder jsonIndex = new JSONIndexBuilder()
137+
.key("author", String.class, JSONIndexBuilder.Order.ASC)
138+
.key("year", Integer.class, JSONIndexBuilder.Order.DESC);
139+
jsonIndex.createOption(CreateOption.CREATE_IF_NOT_EXISTS);
140+
141+
assertThrows(IllegalStateException.class, jsonIndex::build);
142+
}
143+
58144
@Test
59145
void should_create_table_with_ordered_json_index() {
60146
SQLServerDataSource dataSource = getSqlServerDataSource();

0 commit comments

Comments
 (0)