Skip to content

Commit d662f2d

Browse files
clunclun
committed
Override maxChunkSize, maxCount, maxPageSize in dataAPI Options
1 parent a93c335 commit d662f2d

File tree

8 files changed

+149
-61
lines changed

8 files changed

+149
-61
lines changed

‎astra-db-java/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
<uuid-generator.version>5.0.0</uuid-generator.version>
3030

3131
<!-- Test -->
32-
<test.skipped>false</test.skipped>
32+
<test.skipped>true</test.skipped>
3333
<assertj.version>3.25.3</assertj.version>
3434
<junit-jupiter.version>5.10.2</junit-jupiter.version>
3535

‎astra-db-java/src/main/java/com/datastax/astra/client/Collection.java

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,12 @@ public class Collection<T> extends AbstractCommandRunner {
188188
@Getter
189189
private final Database database;
190190

191+
/**
192+
* Get global Settings for the client.
193+
*/
194+
@Getter
195+
private final DataAPIOptions dataAPIOptions;
196+
191197
/** Api Endpoint for the Database, if using an astra environment it will contain the database id and the database region. */
192198
private final String apiEndpoint;
193199

@@ -237,6 +243,7 @@ protected Collection(Database db, String collectionName, Class<T> clazz) {
237243
hasLength(collectionName, ARG_COLLECTION_NAME);
238244
this.collectionName = collectionName;
239245
this.database = db;
246+
this.dataAPIOptions = db.getOptions();
240247
this.documentClass = clazz;
241248
this.apiEndpoint = db.getApiEndpoint() + "/" + collectionName;
242249
}
@@ -735,8 +742,8 @@ public InsertManyResult insertMany(List<? extends T> documents, InsertManyOption
735742
if (options.getConcurrency() > 1 && options.isOrdered()) {
736743
throw new IllegalArgumentException("Cannot run ordered insert_many concurrently.");
737744
}
738-
if (options.getChunkSize() > DataAPIOptions.getMaxDocumentsInInsert()) {
739-
throw new IllegalArgumentException("Cannot insert more than " + DataAPIOptions.getMaxDocumentsInInsert() + " at a time.");
745+
if (options.getChunkSize() > dataAPIOptions.getMaxDocumentsInInsert()) {
746+
throw new IllegalArgumentException("Cannot insert more than " + dataAPIOptions.getMaxDocumentsInInsert() + " at a time.");
740747
}
741748
long start = System.currentTimeMillis();
742749
ExecutorService executor = Executors.newFixedThreadPool(options.getConcurrency());
@@ -1401,8 +1408,8 @@ public int countDocuments(int upperBound) throws TooManyDocumentsToCountExceptio
14011408
public int countDocuments(Filter filter, int upperBound)
14021409
throws TooManyDocumentsToCountException {
14031410
// Argument Validation
1404-
if (upperBound<1 || upperBound> DataAPIOptions.getMaxDocumentCount()) {
1405-
throw new IllegalArgumentException("UpperBound limit should be in between 1 and " + DataAPIOptions.getMaxDocumentCount());
1411+
if (upperBound<1 || upperBound> dataAPIOptions.getMaxDocumentCount()) {
1412+
throw new IllegalArgumentException("UpperBound limit should be in between 1 and " + dataAPIOptions.getMaxDocumentCount());
14061413
}
14071414
// Build command
14081415
Command command = new Command("countDocuments").withFilter(filter);
@@ -1423,8 +1430,6 @@ public int countDocuments(Filter filter, int upperBound)
14231430
// --- Delete ----
14241431
// ----------------------------
14251432

1426-
1427-
14281433
/**
14291434
* Removes at most one document from the collection that matches the given filter.
14301435
* If no documents match, the collection is not modified.

‎astra-db-java/src/main/java/com/datastax/astra/client/DataAPIClient.java

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -100,15 +100,6 @@ public class DataAPIClient {
100100
*/
101101
private final String token;
102102

103-
static {
104-
DataAPIOptions.builder()
105-
.withHttpRequestTimeout(1000)
106-
.withHttpConnectTimeout(10)
107-
.withHtpVersion(HttpClient.Version.HTTP_2)
108-
.withDestination(ASTRA)
109-
.build();
110-
}
111-
112103
/**
113104
* Represents the advanced configuration settings for the client, encapsulating various HTTP-related properties.
114105
* This includes settings such as the HTTP version, request and connection timeouts, and the user agent among others.
@@ -229,7 +220,6 @@ public AstraDBAdmin getAdmin() {
229220
return getAdmin(this.token);
230221
}
231222

232-
233223
/**
234224
* Retrieves an administration client capable of performing CRUD operations on databases, requiring a token with
235225
* advanced privileges. This method is designed for scenarios where administrative access is necessary beyond the
@@ -339,7 +329,7 @@ public Database getDatabase(UUID databaseId, String namespace) {
339329
return new Database(new AstraApiEndpoint(databaseId,
340330
getAdmin().getDatabaseInfo(databaseId).getRegion(),
341331
getAstraEnvironment()).getApiEndPoint(),
342-
this.token, namespace);
332+
this.token, namespace, options);
343333
}
344334

345335
/**
@@ -360,7 +350,7 @@ public Database getDatabase(UUID databaseId, String namespace, String region) {
360350
Assert.hasLength(region, ARG_REGION);
361351
return new Database(new AstraApiEndpoint(databaseId, region,
362352
getAstraEnvironment()).getApiEndPoint(),
363-
this.token, namespace);
353+
this.token, namespace, options);
364354
}
365355

366356
/**

‎astra-db-java/src/main/java/com/datastax/astra/client/DataAPIOptions.java

Lines changed: 121 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -23,23 +23,25 @@
2323
import com.datastax.astra.internal.utils.Assert;
2424
import lombok.Getter;
2525
import lombok.Setter;
26+
import lombok.extern.slf4j.Slf4j;
2627

2728
import java.net.http.HttpClient;
2829

2930
/**
3031
* Options to set up the client for DataApiClient.
3132
*/
33+
@Slf4j
3234
@Getter
3335
public class DataAPIOptions {
3436

3537
/** Number of documents for a count. */
36-
static final int MAX_DOCUMENTS_COUNT = 1000;
38+
public static final int DEFAULT_MAX_DOCUMENTS_COUNT = 1000;
3739

3840
/** Maximum number of documents in a page. */
39-
static final int MAX_PAGE_SIZE = 20;
41+
public static final int DEFAULT_MAX_PAGE_SIZE = 20;
4042

4143
/** Maximum number of documents when you insert. */
42-
static final int MAX_DOCUMENTS_IN_INSERT = 20;
44+
public static final int DEFAULT_MAX_CHUNKSIZE = 20;
4345

4446
/** Default user agent. */
4547
public static final String DEFAULT_CALLER_NAME = "astra-db-java";
@@ -73,6 +75,15 @@ public class DataAPIOptions {
7375
/** Set the API version like 'v1' */
7476
final String apiVersion;
7577

78+
/** When operating a count operation, the maximum number of documents that can be returned. */
79+
final int maxDocumentCount;
80+
81+
/** The maximum number of documents that can be returned in a single page. */
82+
final int maxPageSize;
83+
84+
/** The maximum number of documents that can be inserted in a single operation. */
85+
final int maxDocumentsInInsert;
86+
7687
/**
7788
* Initializer for the builder.
7889
*
@@ -90,9 +101,12 @@ public static DataAPIClientOptionsBuilder builder() {
90101
* current builder
91102
*/
92103
private DataAPIOptions(DataAPIClientOptionsBuilder builder) {
93-
this.apiVersion = builder.apiVersion;
94-
this.destination = builder.destination;
95-
this.httpClientOptions = new HttpClientOptions();
104+
this.apiVersion = builder.apiVersion;
105+
this.destination = builder.destination;
106+
this.maxDocumentCount = builder.maxDocumentCount;
107+
this.maxPageSize = builder.maxPageSize;
108+
this.maxDocumentsInInsert = builder.maxDocumentsInInsert;
109+
this.httpClientOptions = new HttpClientOptions();
96110
httpClientOptions.setHttpVersion(builder.httpVersion);
97111
httpClientOptions.setHttpRedirect(builder.httpRedirect);
98112
httpClientOptions.setRetryCount(builder.retryCount);
@@ -263,6 +277,15 @@ public static class DataAPIClientOptionsBuilder {
263277
/** Default is to use Astra in Production. */
264278
private DataAPIDestination destination = DataAPIDestination.ASTRA;
265279

280+
/** When operating a count operation, the maximum number of documents that can be returned. */
281+
private int maxDocumentCount = DEFAULT_MAX_DOCUMENTS_COUNT;
282+
283+
/** The maximum number of documents that can be returned in a single page. */
284+
private int maxPageSize = DEFAULT_MAX_PAGE_SIZE;
285+
286+
/** The maximum number of documents that can be inserted in a single operation. */
287+
private int maxDocumentsInInsert = DEFAULT_MAX_CHUNKSIZE;
288+
266289
/**
267290
* Default constructor.
268291
*/
@@ -425,6 +448,98 @@ public DataAPIClientOptionsBuilder withHttpRetryDelayMillis(int retryDelay) {
425448
return this;
426449
}
427450

451+
/**
452+
* Sets the maximum number of documents that can be returned by the count function.
453+
* <p>
454+
* If not explicitly set, the default value is defined by {@code MAX_DOCUMENTS_COUNT},
455+
* which is 1000 documents.
456+
* </p>
457+
*
458+
* @param maxDocumentCount the maximum number of documents that can be returned by the count function.
459+
* Must be a positive number.
460+
* @return a reference to this builder, allowing for method chaining.
461+
*
462+
* Example usage:
463+
* <pre>
464+
* {@code
465+
* DataAPIClientOptions
466+
* .builder()
467+
* .withMaxDocumentCount(2000); // Sets the maximum number of documents to 2000.
468+
* }</pre>
469+
*/
470+
public DataAPIClientOptionsBuilder withMaxDocumentCount(int maxDocumentCount) {
471+
if (maxDocumentCount <= 0) {
472+
throw new IllegalArgumentException("Max document count must be a positive number");
473+
}
474+
if (maxDocumentCount > DEFAULT_MAX_DOCUMENTS_COUNT) {
475+
log.warn("Setting the maximum document count to a value greater than the default value of {} may impact performance.", DEFAULT_MAX_DOCUMENTS_COUNT);
476+
}
477+
this.maxDocumentCount = maxDocumentCount;
478+
return this;
479+
}
480+
481+
/**
482+
* Sets the maximum number of documents that can be returned in a single page.
483+
* <p>
484+
* If not explicitly set, the default value is defined by {@code MAX_PAGE_SIZE},
485+
* which is 20 documents.
486+
* </p>
487+
*
488+
* @param maxPageSize the maximum number of documents that can be returned in a single page.
489+
* Must be a positive number.
490+
* @return a reference to this builder, allowing for method chaining.
491+
*
492+
* Example usage:
493+
* <pre>
494+
* {@code
495+
* DataAPIClientOptions
496+
* .builder()
497+
* .withMaxPageSize(50); // Sets the maximum page size to 50 documents.
498+
* }</pre>
499+
*/
500+
public DataAPIClientOptionsBuilder withMaxPageSize(int maxPageSize) {
501+
if (maxPageSize <= 0) {
502+
throw new IllegalArgumentException("Max page size must be a positive number");
503+
}
504+
if (maxPageSize > DEFAULT_MAX_PAGE_SIZE) {
505+
log.warn("Setting the maximum page size to a value greater than the " +
506+
"default value of {} may impact performance or result in error at server level", DEFAULT_MAX_PAGE_SIZE);
507+
}
508+
this.maxPageSize = maxPageSize;
509+
return this;
510+
}
511+
512+
/**
513+
* Sets the maximum number of documents that can be inserted in a single operation.
514+
* <p>
515+
* If not explicitly set, the default value is defined by {@code MAX_DOCUMENTS_IN_INSERT},
516+
* which is 20 documents.
517+
* </p>
518+
*
519+
* @param maxDocumentsInInsert the maximum number of documents that can be inserted in a single operation.
520+
* Must be a positive number.
521+
* @return a reference to this builder, allowing for method chaining.
522+
*
523+
* Example usage:
524+
* <pre>
525+
* {@code
526+
* DataAPIClientOptions
527+
* .builder()
528+
* .withMaxDocumentsInInsert(50); // Sets the maximum number of documents to insert to 50.
529+
* }</pre>
530+
*/
531+
public DataAPIClientOptionsBuilder withMaxDocumentsInInsert(int maxDocumentsInInsert) {
532+
if (maxDocumentsInInsert <= 0) {
533+
throw new IllegalArgumentException("Max documents in insert must be a positive number");
534+
}
535+
if (maxDocumentsInInsert > DEFAULT_MAX_CHUNKSIZE) {
536+
log.warn("Setting the maximum number of documents in insert to a value greater than the " +
537+
"default value of {} may impact performance or result in error at server level", DEFAULT_MAX_CHUNKSIZE);
538+
}
539+
this.maxDocumentsInInsert = maxDocumentsInInsert;
540+
return this;
541+
}
542+
428543
/**
429544
* Build the options.
430545
*
@@ -434,38 +549,6 @@ public DataAPIClientOptionsBuilder withHttpRetryDelayMillis(int retryDelay) {
434549
public DataAPIOptions build() {
435550
return new DataAPIOptions(this);
436551
}
437-
438-
}
439-
440-
441-
/**
442-
* Retrieve the maximum number of documents that the count function can return.
443-
*
444-
* @return
445-
* maximum number of document returned
446-
*/
447-
public static int getMaxDocumentCount() {
448-
return MAX_DOCUMENTS_COUNT;
449-
}
450-
451-
/**
452-
* Retrieve the maximum number of documents for a page and also the maximum you can set for a limit.
453-
*
454-
* @return
455-
* maximum page size.
456-
*/
457-
public static int getMaxPageSize() {
458-
return MAX_PAGE_SIZE;
459-
}
460-
461-
/**
462-
* Retrieve the maximum number of documents allows for a insertMany() below this point the list is split and chunked are processed in parallel.
463-
*
464-
* @return
465-
* maximum page size.
466-
*/
467-
public static int getMaxDocumentsInInsert() {
468-
return MAX_DOCUMENTS_IN_INSERT;
469552
}
470553

471554
}

‎astra-db-java/src/main/java/com/datastax/astra/client/exception/TooManyDocumentsToCountException.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public class TooManyDocumentsToCountException extends Exception {
3131
* Default constructor.
3232
*/
3333
public TooManyDocumentsToCountException() {
34-
super("Document count exceeds '" + DataAPIOptions.getMaxDocumentCount() + ", the maximum allowed by the server");
34+
super("Document count exceeds '" + DataAPIOptions.DEFAULT_MAX_DOCUMENTS_COUNT + ", the maximum allowed by the server");
3535
}
3636

3737
/**

‎astra-db-java/src/main/java/com/datastax/astra/client/model/InsertManyOptions.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public class InsertManyOptions {
4242
/**
4343
* If the flag is set to true the command is failing on first error
4444
*/
45-
private int chunkSize = DataAPIOptions.getMaxDocumentsInInsert();
45+
private int chunkSize = DataAPIOptions.DEFAULT_MAX_CHUNKSIZE;
4646

4747
/**
4848
* If the flag is set to true the command is failing on first error

‎astra-db-java/src/test/java/com/datastax/astra/test/integration/collection/AbstractCollectionITTest.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,7 @@ protected void testCountDocument() throws TooManyDocumentsToCountException {
361361

362362
// Add a filter
363363
assertThat(getCollectionSimple()
364-
.countDocuments(gt("indice", 3), DataAPIOptions.getMaxDocumentCount()))
364+
.countDocuments(gt("indice", 3), getCollectionSimple().getDataAPIOptions().getMaxDocumentCount()) )
365365
.isEqualTo(6);
366366

367367
// Filter + limit
@@ -375,7 +375,7 @@ protected void testCountDocument() throws TooManyDocumentsToCountException {
375375

376376
// More than 1000 items
377377
assertThatThrownBy(() -> getCollectionSimple()
378-
.countDocuments(DataAPIOptions.getMaxDocumentCount()))
378+
.countDocuments(getCollectionSimple().getDataAPIOptions().getMaxDocumentCount()))
379379
.isInstanceOf(TooManyDocumentsToCountException.class)
380380
.hasMessageContaining("server");
381381
}

‎astra-db-java/src/test/java/com/datastax/astra/test/unit/DataApiOptionsTest.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.datastax.astra.test.unit;
22

3+
import com.datastax.astra.client.DataAPIClient;
34
import com.datastax.astra.client.DataAPIOptions;
45
import com.datastax.astra.client.model.CollectionIdTypes;
56
import com.datastax.astra.client.model.CollectionOptions;
@@ -269,5 +270,14 @@ void shouldTestFindOptions() {
269270
assertThat(FindOptions.Builder.skip(10)).isNotNull();
270271
}
271272

273+
@Test
274+
void shouldTOverrideMaximumLimits() {
275+
DataAPIOptions options = DataAPIOptions.builder()
276+
.withMaxDocumentsInInsert(100)
277+
.build();
278+
//DataAPIClient client = new DataAPIClient("token", options);
279+
280+
}
281+
272282

273283
}

0 commit comments

Comments
 (0)