Skip to content

Upgrade pinecone java client to 2.1.0 #1501

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Upgrade pinecone java client to 2.1.0 #1501

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@
<elasticsearch-java.version>8.13.3</elasticsearch-java.version>
<milvus.version>2.3.4</milvus.version>
<gemfire.testcontainers.version>2.3.0</gemfire.testcontainers.version>
<pinecone.version>0.8.0</pinecone.version>
<pinecone.version>2.1.0</pinecone.version>
<fastjson.version>2.0.46</fastjson.version>
<azure-search.version>11.6.1</azure-search.version>
<weaviate-client.version>4.5.1</weaviate-client.version>
Expand Down
127 changes: 29 additions & 98 deletions ...-pinecone-store/src/main/java/org/springframework/ai/vectorstore/PineconeVectorStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

package org.springframework.ai.vectorstore;

import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand All @@ -43,15 +42,11 @@
import com.google.protobuf.util.JsonFormat;

import io.micrometer.observation.ObservationRegistry;
import io.pinecone.PineconeClient;
import io.pinecone.PineconeClientConfig;
import io.pinecone.PineconeConnection;
import io.pinecone.PineconeConnectionConfig;
import io.pinecone.proto.DeleteRequest;
import io.pinecone.proto.QueryRequest;
import io.pinecone.proto.QueryResponse;
import io.pinecone.proto.UpsertRequest;
import io.pinecone.proto.Vector;
import io.pinecone.clients.Index;
import io.pinecone.clients.Pinecone;
import io.pinecone.unsigned_indices_model.QueryResponseWithUnsignedIndices;
import io.pinecone.unsigned_indices_model.VectorWithUnsignedIndices;
import static io.pinecone.commons.IndexInterface.buildUpsertVectorWithUnsignedIndices;

/**
* A VectorStore implementation backed by Pinecone, a cloud-based vector database. This
Expand All @@ -72,7 +67,9 @@ public class PineconeVectorStore extends AbstractObservationVectorStore {

private final EmbeddingModel embeddingModel;

private final PineconeConnection pineconeConnection;
private final Pinecone pinecone;

private final Index index;

private final String pineconeNamespace;

Expand All @@ -91,6 +88,8 @@ public class PineconeVectorStore extends AbstractObservationVectorStore {
*/
public static final class PineconeVectorStoreConfig {

private final String apiKey;

// The free tier (gcp-starter) doesn't support Namespaces.
// Leave the namespace empty (e.g. "") for the free tier.
private final String namespace;
Expand All @@ -99,32 +98,18 @@ public static final class PineconeVectorStoreConfig {

private final String distanceMetadataFieldName;

private final PineconeConnectionConfig connectionConfig;

private final PineconeClientConfig clientConfig;
private final String indexName;

// private final int defaultSimilarityTopK;

/**
* Constructor using the builder.
* @param builder The configuration builder.
*/
/**
* Constructor using the builder.
* @param builder The configuration builder.
*/
public PineconeVectorStoreConfig(Builder builder) {
this.apiKey = builder.apiKey;
this.namespace = builder.namespace;
this.contentFieldName = builder.contentFieldName;
this.distanceMetadataFieldName = builder.distanceMetadataFieldName;

Copy link
Member

@ilayaperumalg ilayaperumalg Nov 11, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jiwhiz Thanks for the PR. Could you explain the reasoning for this change - removal of the some of the existing properties, client configuration and the usage of Index as part of this client version update?

// this.defaultSimilarityTopK = builder.defaultSimilarityTopK;
this.connectionConfig = new PineconeConnectionConfig().withIndexName(builder.indexName);
this.clientConfig = new PineconeClientConfig().withApiKey(builder.apiKey)
.withEnvironment(builder.environment)
.withProjectName(builder.projectId)
.withApiKey(builder.apiKey)
.withServerSideTimeoutSec((int) builder.serverSideTimeout.toSeconds());
this.indexName = builder.indexName;
}

/**
Expand All @@ -146,10 +131,6 @@ public static class Builder {

private String apiKey;

private String projectId;

private String environment;

private String indexName;

// The free-tier (gcp-starter) doesn't support Namespaces!
Expand All @@ -159,12 +140,6 @@ public static class Builder {

private String distanceMetadataFieldName = DISTANCE_METADATA_FIELD_NAME;

/**
* Optional server-side timeout in seconds for all operations. Default: 20
* seconds.
*/
private Duration serverSideTimeout = Duration.ofSeconds(20);

private Builder() {
}

Expand All @@ -178,26 +153,6 @@ public Builder withApiKey(String apiKey) {
return this;
}

/**
* Pinecone project id.
* @param projectId Project id to use.
* @return this builder.
*/
public Builder withProjectId(String projectId) {
this.projectId = projectId;
return this;
}

/**
* Pinecone environment name.
* @param environment Environment name (e.g. gcp-starter).
* @return this builder.
*/
public Builder withEnvironment(String environment) {
this.environment = environment;
return this;
}

/**
* Pinecone index name.
* @param indexName Pinecone index name to use.
Expand Down Expand Up @@ -239,16 +194,6 @@ public Builder withDistanceMetadataFieldName(String distanceMetadataFieldName) {
return this;
}

/**
* Pinecone server side timeout.
* @param serverSideTimeout server timeout to use.
* @return this builder.
*/
public Builder withServerSideTimeout(Duration serverSideTimeout) {
this.serverSideTimeout = serverSideTimeout;
return this;
}

/**
* {@return the immutable configuration}
*/
Expand Down Expand Up @@ -285,10 +230,11 @@ public PineconeVectorStore(PineconeVectorStoreConfig config, EmbeddingModel embe

this.embeddingModel = embeddingModel;
this.pineconeNamespace = config.namespace;
this.pineconeIndexName = config.connectionConfig.getIndexName();
this.pineconeIndexName = config.indexName;
this.pineconeContentFieldName = config.contentFieldName;
this.pineconeDistanceMetadataFieldName = config.distanceMetadataFieldName;
this.pineconeConnection = new PineconeClient(config.clientConfig).connect(config.connectionConfig);
this.pinecone = new Pinecone.Builder(config.apiKey).build();
this.index = pinecone.getIndexConnection(pineconeIndexName);
this.objectMapper = new ObjectMapper();
this.batchingStrategy = batchingStrategy;
}
Expand All @@ -300,20 +246,13 @@ public PineconeVectorStore(PineconeVectorStoreConfig config, EmbeddingModel embe
*/
public void add(List<Document> documents, String namespace) {
this.embeddingModel.embed(documents, EmbeddingOptionsBuilder.builder().build(), this.batchingStrategy);
List<Vector> upsertVectors = documents.stream()
.map(document -> Vector.newBuilder()
.setId(document.getId())
.addAllValues(EmbeddingUtils.toList(document.getEmbedding()))
.setMetadata(metadataToStruct(document))
.build())
.toList();

UpsertRequest upsertRequest = UpsertRequest.newBuilder()
.addAllVectors(upsertVectors)
.setNamespace(namespace)
.build();
List<VectorWithUnsignedIndices> upsertVectors = documents.stream()
.map(document -> buildUpsertVectorWithUnsignedIndices(document.getId(),
EmbeddingUtils.toList(document.getEmbedding()), null, null, metadataToStruct(document)))
.toList();

this.pineconeConnection.getBlockingStub().upsert(upsertRequest);
this.index.upsert(upsertVectors, namespace);
}

/**
Expand Down Expand Up @@ -361,13 +300,7 @@ private Value contentValue(Document document) {
*/
public Optional<Boolean> delete(List<String> documentIds, String namespace) {

DeleteRequest deleteRequest = DeleteRequest.newBuilder()
.setNamespace(namespace) // ignored for free tier.
.addAllIds(documentIds)
.setDeleteAll(false)
.build();

this.pineconeConnection.getBlockingStub().delete(deleteRequest);
this.index.deleteByIds(documentIds, namespace);

// The Pinecone delete API does not provide deletion status info.
return Optional.of(true);
Expand All @@ -390,17 +323,15 @@ public List<Document> similaritySearch(SearchRequest request, String namespace)

float[] queryEmbedding = this.embeddingModel.embed(request.getQuery());

var queryRequestBuilder = QueryRequest.newBuilder()
.addAllVector(EmbeddingUtils.toList(queryEmbedding))
.setTopK(request.getTopK())
.setIncludeMetadata(true)
.setNamespace(namespace);

QueryResponseWithUnsignedIndices queryResponse = null;
if (StringUtils.hasText(nativeExpressionFilters)) {
queryRequestBuilder.setFilter(metadataFiltersToStruct(nativeExpressionFilters));
queryResponse = this.index.queryByVector(request.getTopK(), EmbeddingUtils.toList(queryEmbedding),
namespace, metadataFiltersToStruct(nativeExpressionFilters));
}
else {
queryResponse = this.index.queryByVector(request.getTopK(), EmbeddingUtils.toList(queryEmbedding),
namespace);
}

QueryResponse queryResponse = this.pineconeConnection.getBlockingStub().query(queryRequestBuilder.build());

return queryResponse.getMatchesList()
.stream()
Expand Down
2 changes: 0 additions & 2 deletions ...inecone-store/src/test/java/org/springframework/ai/vectorstore/PineconeVectorStoreIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,6 @@ public PineconeVectorStoreConfig pineconeVectorStoreConfig() {

return PineconeVectorStoreConfig.builder()
.withApiKey(System.getenv("PINECONE_API_KEY"))
.withEnvironment(PINECONE_ENVIRONMENT)
.withProjectId(PINECONE_PROJECT_ID)
.withIndexName(PINECONE_INDEX_NAME)
.withNamespace(PINECONE_NAMESPACE)
.withContentFieldName(CUSTOM_CONTENT_FIELD_NAME)
Expand Down
2 changes: 0 additions & 2 deletions ...re/src/test/java/org/springframework/ai/vectorstore/PineconeVectorStoreObservationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,8 +188,6 @@ public PineconeVectorStoreConfig pineconeVectorStoreConfig() {

return PineconeVectorStoreConfig.builder()
.withApiKey(System.getenv("PINECONE_API_KEY"))
.withEnvironment(PINECONE_ENVIRONMENT)
.withProjectId(PINECONE_PROJECT_ID)
.withIndexName(PINECONE_INDEX_NAME)
.withNamespace(PINECONE_NAMESPACE)
.withContentFieldName(CUSTOM_CONTENT_FIELD_NAME)
Expand Down