diff --git a/embedding-stores/langchain4j-community-typesense/pom.xml b/embedding-stores/langchain4j-community-typesense/pom.xml new file mode 100644 index 00000000..ae46b7fe --- /dev/null +++ b/embedding-stores/langchain4j-community-typesense/pom.xml @@ -0,0 +1,107 @@ + + + 4.0.0 + + dev.langchain4j + langchain4j-community + 1.1.0-beta7-SNAPSHOT + ../../pom.xml + + + langchain4j-community-typesense + LangChain4j :: Community :: Integration :: Typesense + + + + Apache-2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + 1.3.0 + + + + + + dev.langchain4j + langchain4j-core + ${langchain4j.core.version} + + + + org.typesense + typesense-java + ${typesense.version} + + + + org.slf4j + slf4j-api + + + + dev.langchain4j + langchain4j-core + ${langchain4j.core.version} + tests + test-jar + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + + org.junit.jupiter + junit-jupiter-params + test + + + + org.assertj + assertj-core + test + + + + org.tinylog + tinylog-impl + test + + + + org.tinylog + slf4j-tinylog + test + + + + dev.langchain4j + langchain4j-embeddings-all-minilm-l6-v2-q + test + + + + org.testcontainers + typesense + test + + + + org.awaitility + awaitility + test + + + + + diff --git a/embedding-stores/langchain4j-community-typesense/src/main/java/dev/langchain4j/community/store/embedding/typesense/TypesenseEmbeddingStore.java b/embedding-stores/langchain4j-community-typesense/src/main/java/dev/langchain4j/community/store/embedding/typesense/TypesenseEmbeddingStore.java new file mode 100644 index 00000000..37df3779 --- /dev/null +++ b/embedding-stores/langchain4j-community-typesense/src/main/java/dev/langchain4j/community/store/embedding/typesense/TypesenseEmbeddingStore.java @@ -0,0 +1,293 @@ +package dev.langchain4j.community.store.embedding.typesense; + +import static dev.langchain4j.internal.Utils.getOrDefault; +import static dev.langchain4j.internal.Utils.isNullOrEmpty; +import static dev.langchain4j.internal.Utils.randomUUID; +import static dev.langchain4j.internal.ValidationUtils.ensureNotEmpty; +import static dev.langchain4j.internal.ValidationUtils.ensureNotNull; +import static dev.langchain4j.internal.ValidationUtils.ensureTrue; +import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.toList; + +import dev.langchain4j.community.store.embedding.typesense.exception.TypesenseException; +import dev.langchain4j.data.document.Metadata; +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.store.embedding.EmbeddingMatch; +import dev.langchain4j.store.embedding.EmbeddingSearchRequest; +import dev.langchain4j.store.embedding.EmbeddingSearchResult; +import dev.langchain4j.store.embedding.EmbeddingStore; +import dev.langchain4j.store.embedding.filter.Filter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.typesense.api.Client; +import org.typesense.model.CollectionResponse; +import org.typesense.model.DeleteDocumentsParameters; +import org.typesense.model.ImportDocumentsParameters; +import org.typesense.model.IndexAction; +import org.typesense.model.MultiSearchCollectionParameters; +import org.typesense.model.MultiSearchResult; +import org.typesense.model.MultiSearchSearchesParameter; +import org.typesense.model.SearchParameters; +import org.typesense.model.SearchResult; +import org.typesense.model.SearchResultHit; + +/** + * Represents a Typesense embedding store + * + * TODO: javadoc + */ +public class TypesenseEmbeddingStore implements EmbeddingStore { + + private static final Logger log = LoggerFactory.getLogger(TypesenseEmbeddingStore.class); + + private final Client client; + private final TypesenseSchema schema; + private final TypesenseMetadataFilterMapper metadataFilterMapper; + + public TypesenseEmbeddingStore(Client client, TypesenseSchema schema) { + this.client = ensureNotNull(client, "client"); + this.schema = getOrDefault(schema, TypesenseSchema.builder().build()); + this.metadataFilterMapper = new TypesenseMetadataFilterMapper(schema); + + String collectionName = this.schema.getCollectionName(); + try { + if (!collectionExist(collectionName)) { + CollectionResponse response = client.collections().create(this.schema.getCollectionSchema()); + log.debug("Created collection {}, response {}", collectionName, response); + } + } catch (Exception e) { + log.error("Error creating collection {}: {}", collectionName, e.getMessage()); + throw new TypesenseException(String.format("Error creating collection %s", collectionName), e); + } + } + + @Override + public String add(Embedding embedding) { + String id = randomUUID(); + add(id, embedding); + return id; + } + + @Override + public void add(String id, Embedding embedding) { + addInternal(id, embedding, null); + } + + @Override + public String add(Embedding embedding, TextSegment textSegment) { + String id = randomUUID(); + addInternal(id, embedding, textSegment); + return id; + } + + @Override + public List addAll(List embeddings) { + List ids = embeddings.stream().map(ignored -> randomUUID()).collect(toList()); + addAll(ids, embeddings, null); + return ids; + } + + @Override + public void addAll(List ids, List embeddings, List embedded) { + ensureNotEmpty(ids, "ids"); + ensureNotEmpty(embeddings, "embeddings"); + ensureTrue(ids.size() == embeddings.size(), "ids size is not equal to embeddings size"); + ensureTrue( + embedded == null || embeddings.size() == embedded.size(), + "embeddings size is not equal to embedded size"); + + List> documents = new ArrayList<>(); + for (int i = 0; i < ids.size(); i++) { + Map document = new HashMap<>(); + document.put(schema.getIdFieldName(), ids.get(i)); + if (embedded != null) { + TextSegment embeddedText = embedded.get(i); + document.put(schema.getTextFieldName(), embeddedText.text()); + document.put( + schema.getMetadataFieldName(), embeddedText.metadata().toMap()); + } + document.put(schema.getEmbeddingFieldName(), embeddings.get(i).vector()); + documents.add(document); + } + + ImportDocumentsParameters importDocumentsParameters = new ImportDocumentsParameters(); + importDocumentsParameters.action(IndexAction.UPSERT); + + try { + client.collections(schema.getCollectionName()).documents().import_(documents, importDocumentsParameters); + log.info("Added {} documents", documents.size()); + } catch (Exception e) { + log.error("Failed to add documents", e); + throw new TypesenseException("Failed to add documents", e); + } + } + + @Override + public void removeAll(Collection ids) { + ensureNotEmpty(ids, "ids"); + + DeleteDocumentsParameters deleteDocumentsParameters = + new DeleteDocumentsParameters().filterBy(schema.getIdFieldName() + ":=[" + String.join(",", ids) + "]"); + + doRemove(deleteDocumentsParameters); + } + + @Override + public void removeAll(Filter filter) { + ensureNotNull(filter, "filter"); + + String expression = metadataFilterMapper.mapToFilter(filter); + DeleteDocumentsParameters deleteDocumentsParameters = new DeleteDocumentsParameters().filterBy(expression); + + doRemove(deleteDocumentsParameters); + } + + @Override + public void removeAll() { + // Typesense need filter_by to remove + SearchParameters searchParameters = new SearchParameters().q("*").queryBy(schema.getIdFieldName()); + + List ids = new ArrayList<>(); + try { + // FIXME: paginate + SearchResult searchResult = + client.collections(schema.getCollectionName()).documents().search(searchParameters); + Optional.ofNullable(searchResult.getHits()).ifPresent(hit -> hit.stream() + .map(SearchResultHit::getDocument) + .forEach(document -> ids.add((String) document.get(schema.getIdFieldName())))); + } catch (Exception e) { + log.error("Failed to retrieve ids during removeAll", e); + throw new TypesenseException("Failed to retrieve ids during removeAll", e); + } + + String filterBy = schema.getIdFieldName() + ":=[" + String.join(",", ids) + "]"; + doRemove(new DeleteDocumentsParameters().filterBy(filterBy)); + } + + @SuppressWarnings("unchecked") + @Override + public EmbeddingSearchResult search(EmbeddingSearchRequest request) { + String filterExpression = metadataFilterMapper.mapToFilter(request.filter()); + + MultiSearchCollectionParameters multiSearchCollectionParameters = new MultiSearchCollectionParameters(); + multiSearchCollectionParameters.collection(schema.getCollectionName()); + multiSearchCollectionParameters.q("*"); + + String vectorQuery = schema.getEmbeddingFieldName() + ":(" + + "[" + + String.join( + ",", + request.queryEmbedding().vectorAsList().stream() + .map(String::valueOf) + .toList()) + "], " + + "k: " + request.maxResults() + ", " + + "distance_threshold: " + 2 * (1 - request.minScore()) + ")"; + + multiSearchCollectionParameters.vectorQuery(vectorQuery); + if (filterExpression != null) { + multiSearchCollectionParameters.filterBy(filterExpression); + } + MultiSearchSearchesParameter multiSearchesParameter = + new MultiSearchSearchesParameter().addSearchesItem(multiSearchCollectionParameters); + + try { + MultiSearchResult result = client.multiSearch.perform( + multiSearchesParameter, Map.of("query_by", schema.getEmbeddingFieldName())); + + List> results = result.getResults().stream() + .map(item -> { + if (item.getCode() != null) { + log.error("Failed to search documents: {}", item.getError()); + throw new TypesenseException("Failed to search documents: " + item.getError()); + } + + return item.getHits(); + }) + .filter(hits -> !isNullOrEmpty(hits)) + .flatMap(hits -> hits.stream().map(hit -> { + Map rawDocument = hit.getDocument(); + String id = rawDocument.get(schema.getIdFieldName()).toString(); + List embedding = ((List) rawDocument.get(schema.getEmbeddingFieldName())) + .stream().map(Double::floatValue).toList(); + + TextSegment textSegment = null; + if (rawDocument.containsKey(schema.getTextFieldName())) { + Map metadata = + (Map) rawDocument.get(schema.getMetadataFieldName()); + textSegment = TextSegment.from( + (String) rawDocument.get(schema.getTextFieldName()), Metadata.from(metadata)); + } + + // Typesense vector_distance is a value between [0, 2] where 0 represents perfect match and 2 + // represents extreme different + return new EmbeddingMatch<>( + 1 - hit.getVectorDistance().doubleValue() / 2, + id, + Embedding.from(embedding), + textSegment); + })) + .toList(); + + log.info("Found {} documents", results.size()); + + return new EmbeddingSearchResult<>(results); + } catch (Exception e) { + log.error("Failed to search documents", e); + throw new TypesenseException("Failed to search documents", e); + } + } + + private void addInternal(String id, Embedding embedding, TextSegment embedded) { + addAll(singletonList(id), singletonList(embedding), embedded == null ? null : singletonList(embedded)); + } + + private boolean collectionExist(String collectionName) throws Exception { + CollectionResponse[] responses = client.collections().retrieve(); + + return Arrays.stream(responses) + .anyMatch(collection -> collection.getName().equals(collectionName)); + } + + private void doRemove(DeleteDocumentsParameters deleteDocumentsParameters) { + try { + Map res = + client.collections(schema.getCollectionName()).documents().delete(deleteDocumentsParameters); + log.info("Remove {} documents", res.get("num_deleted")); + } catch (Exception e) { + log.error("Failed to remove documents", e); + throw new TypesenseException("Failed to remove documents", e); + } + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private Client client; + private TypesenseSchema schema; + + public Builder client(Client client) { + this.client = client; + return this; + } + + public Builder schema(TypesenseSchema schema) { + this.schema = schema; + return this; + } + + public TypesenseEmbeddingStore build() { + return new TypesenseEmbeddingStore(client, schema); + } + } +} diff --git a/embedding-stores/langchain4j-community-typesense/src/main/java/dev/langchain4j/community/store/embedding/typesense/TypesenseMetadataFilterMapper.java b/embedding-stores/langchain4j-community-typesense/src/main/java/dev/langchain4j/community/store/embedding/typesense/TypesenseMetadataFilterMapper.java new file mode 100644 index 00000000..42899586 --- /dev/null +++ b/embedding-stores/langchain4j-community-typesense/src/main/java/dev/langchain4j/community/store/embedding/typesense/TypesenseMetadataFilterMapper.java @@ -0,0 +1,112 @@ +package dev.langchain4j.community.store.embedding.typesense; + +import dev.langchain4j.store.embedding.filter.Filter; +import dev.langchain4j.store.embedding.filter.comparison.IsEqualTo; +import dev.langchain4j.store.embedding.filter.comparison.IsGreaterThan; +import dev.langchain4j.store.embedding.filter.comparison.IsGreaterThanOrEqualTo; +import dev.langchain4j.store.embedding.filter.comparison.IsIn; +import dev.langchain4j.store.embedding.filter.comparison.IsLessThan; +import dev.langchain4j.store.embedding.filter.comparison.IsLessThanOrEqualTo; +import dev.langchain4j.store.embedding.filter.comparison.IsNotEqualTo; +import dev.langchain4j.store.embedding.filter.comparison.IsNotIn; +import dev.langchain4j.store.embedding.filter.logical.And; +import dev.langchain4j.store.embedding.filter.logical.Or; +import java.util.Arrays; +import java.util.Collection; + +class TypesenseMetadataFilterMapper { + + private final TypesenseSchema schema; + + TypesenseMetadataFilterMapper(TypesenseSchema schema) { + this.schema = schema; + } + + String mapToFilter(Filter filter) { + if (filter == null) { + return null; + } + + if (filter instanceof IsEqualTo isEqualTo) { + return mapEqual(isEqualTo); + } else if (filter instanceof IsNotEqualTo isNotEqualTo) { + return mapNotEqual(isNotEqualTo); + } else if (filter instanceof IsGreaterThan isGreaterThan) { + return mapGreaterThan(isGreaterThan); + } else if (filter instanceof IsGreaterThanOrEqualTo isGreaterThanOrEqualTo) { + return mapGreaterThanOrEqual(isGreaterThanOrEqualTo); + } else if (filter instanceof IsLessThan isLessThan) { + return mapLessThan(isLessThan); + } else if (filter instanceof IsLessThanOrEqualTo isLessThanOrEqualTo) { + return mapLessThanOrEqual(isLessThanOrEqualTo); + } else if (filter instanceof IsIn isIn) { + return mapIn(isIn); + } else if (filter instanceof IsNotIn isNotIn) { + return mapNotIn(isNotIn); + } else if (filter instanceof And and) { + return mapAnd(and); + } else if (filter instanceof Or or) { + return mapOr(or); + } else { + throw new UnsupportedOperationException( + "Unsupported filter type: " + filter.getClass().getName()); + } + } + + private String mapEqual(IsEqualTo filter) { + return String.format("%s:=%s", toKey(filter.key()), toSingleValue(filter.comparisonValue())); + } + + private String mapNotEqual(IsNotEqualTo filter) { + return String.format("%s:!=%s", toKey(filter.key()), toSingleValue(filter.comparisonValue())); + } + + private String mapGreaterThan(IsGreaterThan filter) { + return String.format("%s:>%s", toKey(filter.key()), filter.comparisonValue()); + } + + private String mapGreaterThanOrEqual(IsGreaterThanOrEqualTo filter) { + return String.format("%s:>=%s", toKey(filter.key()), filter.comparisonValue()); + } + + private String mapLessThan(IsLessThan filter) { + return String.format("%s:<%s", toKey(filter.key()), filter.comparisonValue()); + } + + private String mapLessThanOrEqual(IsLessThanOrEqualTo filter) { + return String.format("%s:<=%s", toKey(filter.key()), filter.comparisonValue()); + } + + private String mapIn(IsIn filter) { + return String.format("%s:=%s", toKey(filter.key()), toSingleValue(filter.comparisonValues())); + } + + private String mapNotIn(IsNotIn filter) { + return String.format("%s:!=%s", toKey(filter.key()), toSingleValue(filter.comparisonValues())); + } + + private String mapAnd(And filter) { + return "(" + mapToFilter(filter.left()) + " && " + mapToFilter(filter.right()) + ")"; + } + + private String mapOr(Or filter) { + return "(" + mapToFilter(filter.left()) + " || " + mapToFilter(filter.right()) + ")"; + } + + private String toKey(String rawKey) { + return String.format("%s.%s", schema.getMetadataFieldName(), rawKey); + } + + private String toSingleValue(Object value) { + if (value.getClass().isArray()) { + String[] strings = + Arrays.stream(((Object[]) value)).map(String::valueOf).toArray(String[]::new); + return String.join(", ", strings); + } + if (value instanceof Collection collection) { + return String.join(", ", collection.stream().map(String::valueOf).toArray(String[]::new)); + } + + return value.toString(); + } +} diff --git a/embedding-stores/langchain4j-community-typesense/src/main/java/dev/langchain4j/community/store/embedding/typesense/TypesenseSchema.java b/embedding-stores/langchain4j-community-typesense/src/main/java/dev/langchain4j/community/store/embedding/typesense/TypesenseSchema.java new file mode 100644 index 00000000..45d8e006 --- /dev/null +++ b/embedding-stores/langchain4j-community-typesense/src/main/java/dev/langchain4j/community/store/embedding/typesense/TypesenseSchema.java @@ -0,0 +1,161 @@ +package dev.langchain4j.community.store.embedding.typesense; + +import static dev.langchain4j.internal.ValidationUtils.ensureNotNull; + +import java.util.List; +import org.typesense.api.FieldTypes; +import org.typesense.model.CollectionSchema; +import org.typesense.model.Field; + +public class TypesenseSchema { + + private String idFieldName; + private String textFieldName; + private String embeddingFieldName; + private String metadataFieldName; + private String collectionName; + private CollectionSchema collectionSchema; + + private TypesenseSchema(Builder builder) { + this.idFieldName = ensureNotNull(builder.idFieldName, "idFieldName"); + this.textFieldName = ensureNotNull(builder.textFieldName, "textFieldName"); + this.embeddingFieldName = ensureNotNull(builder.embeddingFieldName, "embeddingFieldName"); + this.metadataFieldName = ensureNotNull(builder.metadataFieldName, "metadataFieldName"); + this.collectionName = ensureNotNull(builder.collectionName, "collectionName"); + this.collectionSchema = builder.collectionSchema; + } + + public String getIdFieldName() { + return idFieldName; + } + + public void setIdFieldName(final String idFieldName) { + this.idFieldName = idFieldName; + } + + public String getTextFieldName() { + return textFieldName; + } + + public void setTextFieldName(final String textFieldName) { + this.textFieldName = textFieldName; + } + + public String getEmbeddingFieldName() { + return embeddingFieldName; + } + + public void setEmbeddingFieldName(final String embeddingFieldName) { + this.embeddingFieldName = embeddingFieldName; + } + + public String getMetadataFieldName() { + return metadataFieldName; + } + + public void setMetadataFieldName(final String metadataFieldName) { + this.metadataFieldName = metadataFieldName; + } + + public String getCollectionName() { + return collectionName; + } + + public void setCollectionName(final String collectionName) { + this.collectionName = collectionName; + } + + public CollectionSchema getCollectionSchema() { + return collectionSchema; + } + + public void setCollectionSchema(final CollectionSchema collectionSchema) { + this.collectionSchema = collectionSchema; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private static final String DEFAULT_ID_FIELD_NAME = "_id"; + private static final String DEFAULT_TEXT_FIELD_NAME = "text"; + private static final String DEFAULT_EMBEDDING_FIELD_NAME = "embedding"; + private static final String DEFAULT_METADATA_FIELD_NAME = "metadata"; + private static final String DEFAULT_COLLECTION_NAME = "langchain4j_collection"; + private static final int DEFAULT_DIMENSION = 384; + private static final CollectionSchema DEFAULT_COLLECTION_SCHEMA = new CollectionSchema(); + + static { + List fields = List.of( + new Field() + .name(DEFAULT_ID_FIELD_NAME) + .type(FieldTypes.STRING) + .optional(false), + new Field() + .name(DEFAULT_TEXT_FIELD_NAME) + .type(FieldTypes.STRING) + .optional(true), + new Field() + .name(DEFAULT_EMBEDDING_FIELD_NAME) + .type(FieldTypes.FLOAT_ARRAY) + .numDim(DEFAULT_DIMENSION) + .optional(false), + new Field() + .name(DEFAULT_METADATA_FIELD_NAME) + .type(FieldTypes.OBJECT) + .optional(true)); + + DEFAULT_COLLECTION_SCHEMA + .name(DEFAULT_COLLECTION_NAME) + .fields(fields) + .enableNestedFields(true); + } + + private String idFieldName = DEFAULT_ID_FIELD_NAME; + private String textFieldName = DEFAULT_TEXT_FIELD_NAME; + private String embeddingFieldName = DEFAULT_EMBEDDING_FIELD_NAME; + private String metadataFieldName = DEFAULT_METADATA_FIELD_NAME; + private String collectionName = DEFAULT_COLLECTION_NAME; + /** + * TODO: schema could be created by field name + */ + private CollectionSchema collectionSchema = DEFAULT_COLLECTION_SCHEMA; + + public Builder idFieldName(String idFieldName) { + this.idFieldName = idFieldName; + return this; + } + + public Builder textFieldName(String textFieldName) { + this.textFieldName = textFieldName; + return this; + } + + public Builder embeddingFieldName(String embeddingFieldName) { + this.embeddingFieldName = embeddingFieldName; + return this; + } + + public Builder metadataFieldName(String metadataFieldName) { + this.metadataFieldName = metadataFieldName; + return this; + } + + public Builder collectionName(String collectionName) { + this.collectionName = collectionName; + this.collectionSchema.name(collectionName); + return this; + } + + public Builder collectionSchema(CollectionSchema collectionSchema) { + this.collectionSchema = collectionSchema; + return this; + } + + public TypesenseSchema build() { + return new TypesenseSchema(this); + } + } +} diff --git a/embedding-stores/langchain4j-community-typesense/src/main/java/dev/langchain4j/community/store/embedding/typesense/exception/TypesenseException.java b/embedding-stores/langchain4j-community-typesense/src/main/java/dev/langchain4j/community/store/embedding/typesense/exception/TypesenseException.java new file mode 100644 index 00000000..43ab8a41 --- /dev/null +++ b/embedding-stores/langchain4j-community-typesense/src/main/java/dev/langchain4j/community/store/embedding/typesense/exception/TypesenseException.java @@ -0,0 +1,14 @@ +package dev.langchain4j.community.store.embedding.typesense.exception; + +import dev.langchain4j.exception.LangChain4jException; + +public class TypesenseException extends LangChain4jException { + + public TypesenseException(String message) { + super(message); + } + + public TypesenseException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/embedding-stores/langchain4j-community-typesense/src/test/java/com/langchain4j/community/store/embedding/typesense/TypesenseEmbeddingStoreIT.java b/embedding-stores/langchain4j-community-typesense/src/test/java/com/langchain4j/community/store/embedding/typesense/TypesenseEmbeddingStoreIT.java new file mode 100644 index 00000000..44670a49 --- /dev/null +++ b/embedding-stores/langchain4j-community-typesense/src/test/java/com/langchain4j/community/store/embedding/typesense/TypesenseEmbeddingStoreIT.java @@ -0,0 +1,57 @@ +package com.langchain4j.community.store.embedding.typesense; + +import dev.langchain4j.community.store.embedding.typesense.TypesenseEmbeddingStore; +import dev.langchain4j.community.store.embedding.typesense.TypesenseSchema; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingStore; +import dev.langchain4j.store.embedding.EmbeddingStoreIT; +import java.time.Duration; +import java.util.List; +import java.util.Random; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.typesense.TypesenseContainer; +import org.typesense.api.Client; +import org.typesense.api.Configuration; +import org.typesense.resources.Node; + +class TypesenseEmbeddingStoreIT extends EmbeddingStoreIT { + + static TypesenseContainer typesense = new TypesenseContainer("typesense/typesense:28.0"); + + Random random = new Random(); + + EmbeddingStore embeddingStore = TypesenseEmbeddingStore.builder() + .client(new Client(new Configuration( + List.of(new Node("http", typesense.getHost(), typesense.getHttpPort())), + Duration.ofSeconds(2), + typesense.getApiKey()))) + .schema(TypesenseSchema.builder() + .collectionName("langchain4j_collection_" + random.nextInt(10000)) + .build()) + .build(); + + EmbeddingModel embeddingModel = new AllMiniLmL6V2QuantizedEmbeddingModel(); + + @BeforeAll + static void beforeAll() { + typesense.start(); + } + + @AfterAll + static void afterAll() { + typesense.stop(); + } + + @Override + protected EmbeddingStore embeddingStore() { + return embeddingStore; + } + + @Override + protected EmbeddingModel embeddingModel() { + return embeddingModel; + } +} diff --git a/embedding-stores/langchain4j-community-typesense/src/test/java/com/langchain4j/community/store/embedding/typesense/TypesenseEmbeddingStoreWithRemovalIT.java b/embedding-stores/langchain4j-community-typesense/src/test/java/com/langchain4j/community/store/embedding/typesense/TypesenseEmbeddingStoreWithRemovalIT.java new file mode 100644 index 00000000..f5cdb5f3 --- /dev/null +++ b/embedding-stores/langchain4j-community-typesense/src/test/java/com/langchain4j/community/store/embedding/typesense/TypesenseEmbeddingStoreWithRemovalIT.java @@ -0,0 +1,62 @@ +package com.langchain4j.community.store.embedding.typesense; + +import dev.langchain4j.community.store.embedding.typesense.TypesenseEmbeddingStore; +import dev.langchain4j.community.store.embedding.typesense.TypesenseSchema; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingStore; +import dev.langchain4j.store.embedding.EmbeddingStoreWithRemovalIT; +import java.time.Duration; +import java.util.List; +import java.util.Random; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.typesense.TypesenseContainer; +import org.typesense.api.Client; +import org.typesense.api.Configuration; +import org.typesense.resources.Node; + +class TypesenseEmbeddingStoreWithRemovalIT extends EmbeddingStoreWithRemovalIT { + + static TypesenseContainer typesense = new TypesenseContainer("typesense/typesense:28.0"); + + Random random = new Random(); + + EmbeddingStore embeddingStore = TypesenseEmbeddingStore.builder() + .client(new Client(new Configuration( + List.of(new Node("http", typesense.getHost(), typesense.getHttpPort())), + Duration.ofSeconds(2), + typesense.getApiKey()))) + .schema(TypesenseSchema.builder() + .collectionName("langchain4j_collection_" + random.nextInt(10000)) + .build()) + .build(); + + EmbeddingModel embeddingModel = new AllMiniLmL6V2QuantizedEmbeddingModel(); + + @BeforeAll + static void beforeAll() { + typesense.start(); + } + + @AfterAll + static void afterAll() { + typesense.stop(); + } + + @Override + protected EmbeddingStore embeddingStore() { + return embeddingStore; + } + + @Override + protected EmbeddingModel embeddingModel() { + return embeddingModel; + } + + @Override + protected boolean supportsRemoveAllByFilter() { + return false; + } +} diff --git a/langchain4j-community-bom/pom.xml b/langchain4j-community-bom/pom.xml index 7b20b84f..0fa3fb32 100644 --- a/langchain4j-community-bom/pom.xml +++ b/langchain4j-community-bom/pom.xml @@ -63,25 +63,25 @@ dev.langchain4j - langchain4j-community-clickhouse + langchain4j-community-alloydb-pg ${project.version} dev.langchain4j - langchain4j-community-duckdb + langchain4j-community-clickhouse ${project.version} dev.langchain4j - langchain4j-community-redis + langchain4j-community-cloud-sql-pg ${project.version} dev.langchain4j - langchain4j-community-vearch + langchain4j-community-duckdb ${project.version} @@ -93,13 +93,19 @@ dev.langchain4j - langchain4j-community-alloydb-pg + langchain4j-community-redis ${project.version} dev.langchain4j - langchain4j-community-cloud-sql-pg + langchain4j-community-typesense + ${project.version} + + + + dev.langchain4j + langchain4j-community-vearch ${project.version} diff --git a/pom.xml b/pom.xml index a23bd5cd..0236811e 100644 --- a/pom.xml +++ b/pom.xml @@ -41,13 +41,14 @@ models/langchain4j-community-zhipu-ai + embedding-stores/langchain4j-community-alloydb-pg embedding-stores/langchain4j-community-clickhouse + embedding-stores/langchain4j-community-cloud-sql-pg embedding-stores/langchain4j-community-duckdb + embedding-stores/langchain4j-community-neo4j embedding-stores/langchain4j-community-redis + embedding-stores/langchain4j-community-typesense embedding-stores/langchain4j-community-vearch - embedding-stores/langchain4j-community-alloydb-pg - embedding-stores/langchain4j-community-cloud-sql-pg - embedding-stores/langchain4j-community-neo4j content-retrievers/langchain4j-community-lucene