Skip to content

Commit 23b7c63

Browse files
clunclun
committed
test
1 parent b63b035 commit 23b7c63

File tree

5 files changed

+195
-15
lines changed

5 files changed

+195
-15
lines changed

‎.github/workflows/ci-astra-dev-vectorize-hf-dedicated.yml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: CI for AstraDEV
2+
3+
on:
4+
#push:
5+
# branches: [ main ]
6+
#pull_request:
7+
# branches: [ main ]
8+
workflow_dispatch:
9+
10+
jobs:
11+
BUILD:
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Checkout Code
15+
uses: actions/checkout@v2
16+
- name: Set up JDK 11
17+
uses: actions/setup-java@v2
18+
with:
19+
distribution: 'adopt'
20+
java-version: 11
21+
- name: Build with Maven
22+
run: |
23+
mvn install -Dmaven.test.skip=true
24+
25+
AWS_HF_DEDICATED:
26+
needs: BUILD
27+
runs-on: ubuntu-latest
28+
steps:
29+
- name: Checkout repository
30+
uses: actions/checkout@v2
31+
- name: Set up JDK
32+
uses: actions/setup-java@v2
33+
with:
34+
distribution: 'adopt'
35+
java-version: 11
36+
- name: Maven Test
37+
env:
38+
ASTRA_DB_APPLICATION_TOKEN_DEV: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN_DEV }}
39+
ASTRA_CLOUD_PROVIDER_DEV: AWS
40+
ASTRA_CLOUD_REGION_DEV: us-west-2
41+
EMBEDDING_PROVIDER: huggingfaceDedicated
42+
EMBEDDING_API_KEY: ${{ secrets.HUGGINGFACEDED_API_KEY }}
43+
HUGGINGFACEDED_DIMENSION: ${{ env.HUGGINGFACEDED_DIMENSION }}
44+
HUGGINGFACEDED_ENDPOINTNAME: ${{ env.HUGGINGFACEDED_ENDPOINTNAME }}
45+
HUGGINGFACEDED_REGIONNAME: ${{ env.HUGGINGFACEDED_REGIONNAME }}
46+
HUGGINGFACEDED_CLOUDNAME: ${{ env.HUGGINGFACEDED_CLOUDNAME }}
47+
run: |
48+
cd astra-db-java
49+
mvn test -Dtest=com.datastax.astra.test.integration.dev_vectorize.AstraDevVectorizeHuggingFaceDedicatedITTest

‎.github/workflows/ci-astra-dev.yml

Lines changed: 65 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
name: CI for AstraDEV
22

33
on:
4-
push:
5-
branches: [ main ]
6-
pull_request:
7-
branches: [ main ]
4+
#push:
5+
# branches: [ main ]
6+
#pull_request:
7+
# branches: [ main ]
88
workflow_dispatch:
99

1010
jobs:
@@ -168,8 +168,52 @@ jobs:
168168
run: |
169169
cd astra-db-java
170170
mvn test -Dtest=com.datastax.astra.test.integration.dev_vectorize.AstraDevVectorizeEmbeddingApiKeyITTest
171+
AWS_VOYAGE_AI:
172+
needs: AWS_OPEN_AI
173+
runs-on: ubuntu-latest
174+
steps:
175+
- name: Checkout repository
176+
uses: actions/checkout@v2
177+
- name: Set up JDK
178+
uses: actions/setup-java@v2
179+
with:
180+
distribution: 'adopt'
181+
java-version: 11
182+
- name: Maven Test
183+
env:
184+
ASTRA_DB_APPLICATION_TOKEN_DEV: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN_DEV }}
185+
ASTRA_CLOUD_PROVIDER_DEV: AWS
186+
ASTRA_CLOUD_REGION_DEV: us-west-2
187+
EMBEDDING_PROVIDER: voyageAI
188+
EMBEDDING_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
189+
run: |
190+
cd astra-db-java
191+
mvn test -Dtest=com.datastax.astra.test.integration.dev_vectorize.AstraDevVectorizeEmbeddingApiKeyITTest
192+
AWS_UPSTAGE_AI:
193+
needs: AWS_OPEN_AI
194+
runs-on: ubuntu-latest
195+
steps:
196+
- name: Checkout repository
197+
uses: actions/checkout@v2
198+
- name: Set up JDK
199+
uses: actions/setup-java@v2
200+
with:
201+
distribution: 'adopt'
202+
java-version: 11
203+
- name: Maven Test
204+
env:
205+
ASTRA_DB_APPLICATION_TOKEN_DEV: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN_DEV }}
206+
ASTRA_CLOUD_PROVIDER_DEV: AWS
207+
ASTRA_CLOUD_REGION_DEV: us-west-2
208+
EMBEDDING_PROVIDER: upstageAI
209+
EMBEDDING_API_KEY: ${{ secrets.UPSTAGE_API_KEY }}
210+
run: |
211+
cd astra-db-java
212+
mvn test -Dtest=com.datastax.astra.test.integration.dev_vectorize.AstraDevVectorizeEmbeddingApiKeyITTest
213+
214+
171215
AWS_JINA_AI:
172-
needs: AWS_AZURE_OPEN_AI
216+
needs: AWS_UPSTAGE_AI
173217
runs-on: ubuntu-latest
174218
steps:
175219
- name: Checkout repository
@@ -191,8 +235,8 @@ jobs:
191235
mvn test -Dtest=com.datastax.astra.test.integration.dev_vectorize.AstraDevVectorizeEmbeddingApiKeyITTest
192236
193237

194-
AWS_VOYAGE_AI:
195-
needs: AWS_OPEN_AI
238+
AWS_NVIDIA:
239+
needs: AWS_VOYAGE_AI
196240
runs-on: ubuntu-latest
197241
steps:
198242
- name: Checkout repository
@@ -207,15 +251,17 @@ jobs:
207251
ASTRA_DB_APPLICATION_TOKEN_DEV: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN_DEV }}
208252
ASTRA_CLOUD_PROVIDER_DEV: AWS
209253
ASTRA_CLOUD_REGION_DEV: us-west-2
210-
EMBEDDING_PROVIDER: voyageAI
211-
EMBEDDING_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
254+
EMBEDDING_PROVIDER: nvidia
255+
EMBEDDING_API_KEY: nvidia # required for github actions, not used
212256
run: |
213257
cd astra-db-java
214258
mvn test -Dtest=com.datastax.astra.test.integration.dev_vectorize.AstraDevVectorizeEmbeddingApiKeyITTest
215259
216260

217-
AWS_UPSTAGE_AI:
218-
needs: AWS_OPEN_AI
261+
262+
263+
AWS_HF_DEDICATED:
264+
needs: AWS_NVIDIA
219265
runs-on: ubuntu-latest
220266
steps:
221267
- name: Checkout repository
@@ -230,12 +276,18 @@ jobs:
230276
ASTRA_DB_APPLICATION_TOKEN_DEV: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN_DEV }}
231277
ASTRA_CLOUD_PROVIDER_DEV: AWS
232278
ASTRA_CLOUD_REGION_DEV: us-west-2
233-
EMBEDDING_PROVIDER: upstageAI
234-
EMBEDDING_API_KEY: ${{ secrets.UPSTAGE_API_KEY }}
279+
EMBEDDING_PROVIDER: huggingfaceDedicated
280+
EMBEDDING_API_KEY: ${{ secrets.HUGGINGFACEDED_API_KEY }}
281+
HUGGINGFACEDED_DIMENSION: ${{ env.HUGGINGFACEDED_DIMENSION }}
282+
HUGGINGFACEDED_ENDPOINTNAME: ${{ env.HUGGINGFACEDED_ENDPOINTNAME }}
283+
HUGGINGFACEDED_REGIONNAME: ${{ env.HUGGINGFACEDED_REGIONNAME }}
284+
HUGGINGFACEDED_CLOUDNAME: ${{ env.HUGGINGFACEDED_CLOUDNAME }}
235285
run: |
236286
cd astra-db-java
237287
mvn test -Dtest=com.datastax.astra.test.integration.dev_vectorize.AstraDevVectorizeEmbeddingApiKeyITTest
238288
289+
290+
239291

240292
GCP_OPEN_AI:
241293
needs: GCP_EUROPE_WEST4

‎astra-db-java/src/test/java/com/datastax/astra/test/integration/AbstractVectorizeITTest.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -267,13 +267,14 @@ public void shouldTestOneProviderSharedKey(String provider, String keyName) {
267267
public void testEmbeddingProvider(String key, EmbeddingProvider provider, String apiKey) {
268268
System.out.println("TESTING PROVIDER [" + key + "]");
269269
Map<String, Object> params = getParameters(key);
270+
final String targetApiKey = ("nvidia".equals(key)) ? null : apiKey;
270271
provider.getModels().forEach(model -> {
271272
System.out.println("Processing MODEL " + model.getName());
272273
try {
273274
log.info("Testing model {}", model);
274-
Collection<Document> collection = createCollectionHeader(key, model, apiKey, params);
275+
Collection<Document> collection = createCollectionHeader(key, model, targetApiKey, params);
275276
log.info("Collection created {}", collection.getName());
276-
testCollection(collection, new EmbeddingAPIKeyHeaderProvider(apiKey));
277+
testCollection(collection, new EmbeddingAPIKeyHeaderProvider(targetApiKey));
277278
collection.drop();
278279

279280
} catch(Exception e) {

‎astra-db-java/src/test/java/com/datastax/astra/test/integration/dev_vectorize/AstraDevVectorizeAwsBedRockITTest.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
package com.datastax.astra.test.integration.dev_vectorize;
2+
3+
public class AstraDevVectorizeAwsBedRockITTest {
4+
}

‎astra-db-java/src/test/java/com/datastax/astra/test/integration/dev_vectorize/AstraDevVectorizeHuggingFaceDedicatedITTest.java

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
package com.datastax.astra.test.integration.dev_vectorize;
2+
3+
import com.datastax.astra.client.Collection;
4+
import com.datastax.astra.client.Database;
5+
import com.datastax.astra.client.auth.EmbeddingAPIKeyHeaderProvider;
6+
import com.datastax.astra.client.model.CollectionOptions;
7+
import com.datastax.astra.client.model.CommandOptions;
8+
import com.datastax.astra.client.model.Document;
9+
import com.datastax.astra.client.model.FindEmbeddingProvidersResult;
10+
import com.datastax.astra.client.model.SimilarityMetric;
11+
import com.datastax.astra.test.integration.AbstractVectorizeITTest;
12+
import com.dtsx.astra.sdk.db.domain.CloudProviderType;
13+
import com.dtsx.astra.sdk.utils.AstraEnvironment;
14+
import lombok.extern.slf4j.Slf4j;
15+
import org.junit.jupiter.api.Test;
16+
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
17+
18+
import java.util.HashMap;
19+
import java.util.Map;
20+
21+
import static org.assertj.core.api.Assertions.assertThat;
22+
23+
@Slf4j
24+
@EnabledIfEnvironmentVariable(named = "ASTRA_DB_APPLICATION_TOKEN_DEV", matches = "Astra.*")
25+
@EnabledIfEnvironmentVariable(named = "ASTRA_CLOUD_PROVIDER_DEV", matches = ".*")
26+
@EnabledIfEnvironmentVariable(named = "ASTRA_CLOUD_REGION_DEV", matches = ".*")
27+
@EnabledIfEnvironmentVariable(named = "EMBEDDING_API_KEY", matches = ".*")
28+
@EnabledIfEnvironmentVariable(named = "EMBEDDING_PROVIDER", matches = ".*")
29+
public class AstraDevVectorizeHuggingFaceDedicatedITTest extends AbstractVectorizeITTest {
30+
31+
@Override
32+
public AstraEnvironment getAstraEnvironment() {
33+
return AstraEnvironment.DEV;
34+
}
35+
36+
@Override
37+
public CloudProviderType getCloudProvider() {
38+
return CloudProviderType.valueOf(System.getenv("ASTRA_CLOUD_PROVIDER_DEV"));
39+
}
40+
41+
@Override
42+
public String getRegion() {
43+
return System.getenv("ASTRA_CLOUD_REGION_DEV");
44+
}
45+
46+
@Test
47+
public void testHuggingFaceDedicated() {
48+
String providerName = System.getenv("EMBEDDING_PROVIDER");
49+
String huggingFaceKey = System.getenv("EMBEDDING_API_KEY");
50+
String collectionName = "collection_hf_dedicated";
51+
FindEmbeddingProvidersResult result = getDatabaseAdmin().findEmbeddingProviders();
52+
assertThat(result.getEmbeddingProviders().get(providerName)).isNotNull();
53+
54+
// Create Collection
55+
CollectionOptions.CollectionOptionsBuilder builder = CollectionOptions.builder();
56+
builder.vectorDimension(Integer.valueOf(System.getenv("HUGGINGFACEDED_DIMENSION")));
57+
Map<String, Object > params = new HashMap<>();
58+
params.put("endpointName", System.getenv("HUGGINGFACEDED_ENDPOINTNAME"));
59+
params.put("regionName", System.getenv("HUGGINGFACEDED_REGIONNAME"));
60+
params.put("cloudName", System.getenv("HUGGINGFACEDED_CLOUDNAME"));
61+
builder.vectorize(providerName, null, huggingFaceKey, params);
62+
Collection<Document> collection = getDatabase()
63+
.createCollection(collectionName, builder.build(), new CommandOptions<>());
64+
assertThat(getDatabase()
65+
.collectionExists(collectionName)).isTrue();
66+
67+
// Test Collection
68+
testCollection(collection, new EmbeddingAPIKeyHeaderProvider(huggingFaceKey));
69+
70+
// Drop Collection
71+
getDatabase().dropCollection(collectionName);
72+
}
73+
74+
}

0 commit comments

Comments
 (0)