Skip to content

Commit 8ef9562

Browse files
jmazanec15jmazanec15
authored
Enhance derived source its (opensearch-project#2648)
Signed-off-by: John Mazanec <jmazane@amazon.com>
1 parent 7157d67 commit 8ef9562

File tree

4 files changed

+208
-21
lines changed

4 files changed

+208
-21
lines changed

‎build.gradle‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -513,7 +513,7 @@ def commonIntegTestClusters(OpenSearchCluster cluster, _numNodes){
513513

514514

515515
// Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
516-
if (_numNodes > 1) numberOfNodes = _numNodes
516+
if (_numNodes > 1) cluster.numberOfNodes = _numNodes
517517
// When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
518518
// i.e. we have to use a custom property to flag when we want to debug opensearch JVM
519519
// since we also support multi node integration tests we increase debugPort per node
@@ -559,6 +559,7 @@ task integTestRemote(type: RestIntegTestTask) {
559559

560560
systemProperty 'tests.security.manager', 'false'
561561
systemProperty("test.exhaustive", System.getProperty("test.exhaustive"))
562+
systemProperty "tests.path.repo", "${layout.buildDirectory.toString()}/testSnapshotFolder"
562563

563564
// Run tests with remote cluster only if rest case is defined
564565
if (System.getProperty("tests.rest.cluster") != null) {

‎src/test/java/org/opensearch/knn/integ/DerivedSourceIT.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,16 @@
66
package org.opensearch.knn.integ;
77

88
import lombok.SneakyThrows;
9+
import org.junit.Before;
10+
import org.opensearch.common.settings.Settings;
911
import org.opensearch.knn.DerivedSourceTestCase;
1012
import org.opensearch.knn.DerivedSourceUtils;
1113
import org.opensearch.knn.Pair;
1214
import org.opensearch.knn.index.VectorDataType;
1315

1416
import java.util.ArrayList;
1517
import java.util.List;
18+
import java.util.Locale;
1619
import java.util.Random;
1720

1821
import static org.opensearch.knn.DerivedSourceUtils.DERIVED_ENABLED_WITH_SEGREP_SETTINGS;
@@ -25,6 +28,18 @@
2528
*/
2629
public class DerivedSourceIT extends DerivedSourceTestCase {
2730

31+
private final String snapshot = "snapshot-test";
32+
private final String repository = "repo";
33+
34+
@Before
35+
@SneakyThrows
36+
public void setUp() {
37+
super.setUp();
38+
final String pathRepo = System.getProperty("tests.path.repo");
39+
Settings repoSettings = Settings.builder().put("compress", randomBoolean()).put("location", pathRepo).build();
40+
registerRepository(repository, "fs", true, repoSettings);
41+
}
42+
2843
@SneakyThrows
2944
public void testFlatFields() {
3045
List<DerivedSourceUtils.IndexConfigContext> indexConfigContexts = getFlatIndexContexts("derivedit", true, true);
@@ -144,6 +159,8 @@ private void testDerivedSourceE2E(List<DerivedSourceUtils.IndexConfigContext> in
144159

145160
// Reindex
146161
testReindex(indexConfigContexts);
147-
}
148162

163+
// Snapshot restore
164+
testSnapshotRestore(repository, snapshot + getTestName().toLowerCase(Locale.ROOT), indexConfigContexts);
165+
}
149166
}

‎src/testFixtures/java/org/opensearch/knn/DerivedSourceTestCase.java‎

Lines changed: 160 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,9 @@ public class DerivedSourceTestCase extends KNNRestTestCase {
3535
);
3636

3737
private static final int MIN_DIMENSION = 4;
38-
private static final int MAX_DIMENSION = 64;
39-
private static final int MIN_DOCS = 100;
40-
private static final int MAX_DOCS = 500;
38+
private static final int MAX_DIMENSION = 32;
39+
private static final int MIN_DOCS = 50;
40+
private static final int MAX_DOCS = 200;
4141

4242
/**
4343
* Testing flat, single field base case with index configuration. The test will automatically skip adding fields for
@@ -164,11 +164,11 @@ protected List<DerivedSourceUtils.IndexConfigContext> getFlatIndexContexts(Strin
164164
* "properties" : {
165165
* "test_vector" : {
166166
* "type" : "knn_vector",
167-
* "dimension" : 16
167+
* "dimension" : 63
168168
* },
169169
* "update_vector" : {
170170
* "type" : "knn_vector",
171-
* "dimension" : 16
171+
* "dimension" : 34
172172
* }
173173
* }
174174
* },
@@ -181,11 +181,11 @@ protected List<DerivedSourceUtils.IndexConfigContext> getFlatIndexContexts(Strin
181181
* },
182182
* "test_vector" : {
183183
* "type" : "knn_vector",
184-
* "dimension" : 16
184+
* "dimension" : 41
185185
* },
186186
* "update_vector" : {
187187
* "type" : "knn_vector",
188-
* "dimension" : 16
188+
* "dimension" : 8
189189
* }
190190
* }
191191
* },
@@ -194,11 +194,11 @@ protected List<DerivedSourceUtils.IndexConfigContext> getFlatIndexContexts(Strin
194194
* },
195195
* "test_vector" : {
196196
* "type" : "knn_vector",
197-
* "dimension" : 16
197+
* "dimension" : 45
198198
* },
199199
* "update_vector" : {
200200
* "type" : "knn_vector",
201-
* "dimension" : 16
201+
* "dimension" : 7
202202
* }
203203
* }
204204
* },
@@ -210,11 +210,11 @@ protected List<DerivedSourceUtils.IndexConfigContext> getFlatIndexContexts(Strin
210210
* },
211211
* "test_vector" : {
212212
* "type" : "knn_vector",
213-
* "dimension" : 16
213+
* "dimension" : 10
214214
* },
215215
* "update_vector" : {
216216
* "type" : "knn_vector",
217-
* "dimension" : 16
217+
* "dimension" : 51
218218
* }
219219
* }
220220
* }
@@ -318,13 +318,24 @@ protected List<DerivedSourceUtils.IndexConfigContext> getObjectIndexContexts(Str
318318
* "nested_1" : {
319319
* "type" : "nested",
320320
* "properties" : {
321+
* "object_1" : {
322+
* "properties" : {
323+
* "test-int" : {
324+
* "type" : "integer"
325+
* },
326+
* "test_vector" : {
327+
* "type" : "knn_vector",
328+
* "dimension" : 64
329+
* }
330+
* }
331+
* },
321332
* "test_vector" : {
322333
* "type" : "knn_vector",
323-
* "dimension" : 16
334+
* "dimension" : 9
324335
* },
325336
* "update_vector" : {
326337
* "type" : "knn_vector",
327-
* "dimension" : 16
338+
* "dimension" : 4
328339
* }
329340
* }
330341
* },
@@ -339,11 +350,11 @@ protected List<DerivedSourceUtils.IndexConfigContext> getObjectIndexContexts(Str
339350
* },
340351
* "test_vector" : {
341352
* "type" : "knn_vector",
342-
* "dimension" : 16
353+
* "dimension" : 27
343354
* },
344355
* "update_vector" : {
345356
* "type" : "knn_vector",
346-
* "dimension" : 16
357+
* "dimension" : 14
347358
* }
348359
* }
349360
* },
@@ -352,11 +363,28 @@ protected List<DerivedSourceUtils.IndexConfigContext> getObjectIndexContexts(Str
352363
* },
353364
* "test_vector" : {
354365
* "type" : "knn_vector",
355-
* "dimension" : 16
366+
* "dimension" : 57
356367
* },
357368
* "update_vector" : {
358369
* "type" : "knn_vector",
359-
* "dimension" : 16
370+
* "dimension" : 10
371+
* }
372+
* }
373+
* },
374+
* "object_1" : {
375+
* "properties" : {
376+
* "nested_1" : {
377+
* "type" : "nested",
378+
* "properties" : {
379+
* "test_vector" : {
380+
* "type" : "knn_vector",
381+
* "dimension" : 30
382+
* }
383+
* }
384+
* },
385+
* "test_vector" : {
386+
* "type" : "knn_vector",
387+
* "dimension" : 51
360388
* }
361389
* }
362390
* },
@@ -368,11 +396,11 @@ protected List<DerivedSourceUtils.IndexConfigContext> getObjectIndexContexts(Str
368396
* },
369397
* "test_vector" : {
370398
* "type" : "knn_vector",
371-
* "dimension" : 16
399+
* "dimension" : 63
372400
* },
373401
* "update_vector" : {
374402
* "type" : "knn_vector",
375-
* "dimension" : 16
403+
* "dimension" : 4
376404
* }
377405
* }
378406
* }
@@ -391,6 +419,28 @@ protected List<DerivedSourceUtils.IndexConfigContext> getNestedIndexContexts(Str
391419
.random(new Random(consistentRandomSeed))
392420
.fields(
393421
List.of(
422+
DerivedSourceUtils.ObjectFieldContext.builder()
423+
.fieldPath("object_1")
424+
.children(
425+
List.of(
426+
DerivedSourceUtils.KNNVectorFieldTypeContext.builder()
427+
.dimension(dimensionSupplier.get())
428+
.fieldPath("object_1.test_vector")
429+
.build(),
430+
DerivedSourceUtils.NestedFieldContext.builder()
431+
.fieldPath("object_1.nested_1")
432+
.children(
433+
List.of(
434+
DerivedSourceUtils.KNNVectorFieldTypeContext.builder()
435+
.dimension(dimensionSupplier.get())
436+
.fieldPath("object_1.nested_1.test_vector")
437+
.build()
438+
)
439+
)
440+
.build()
441+
)
442+
)
443+
.build(),
394444
DerivedSourceUtils.NestedFieldContext.builder()
395445
.fieldPath("nested_1")
396446
.children(
@@ -403,6 +453,18 @@ protected List<DerivedSourceUtils.IndexConfigContext> getNestedIndexContexts(Str
403453
.dimension(dimensionSupplier.get())
404454
.fieldPath("nested_1.update_vector")
405455
.isUpdate(true)
456+
.build(),
457+
DerivedSourceUtils.ObjectFieldContext.builder()
458+
.fieldPath("nested_1.object_1")
459+
.children(
460+
List.of(
461+
DerivedSourceUtils.KNNVectorFieldTypeContext.builder()
462+
.dimension(dimensionSupplier.get())
463+
.fieldPath("nested_1.object_1.test_vector")
464+
.build(),
465+
DerivedSourceUtils.IntFieldType.builder().fieldPath("nested_1.object_1.test-int").build()
466+
)
467+
)
406468
.build()
407469
)
408470
)
@@ -755,6 +817,85 @@ protected void testReindex(List<DerivedSourceUtils.IndexConfigContext> indexConf
755817
);
756818
}
757819

820+
@SneakyThrows
821+
protected void testSnapshotRestore(
822+
String repository,
823+
String snapshot,
824+
List<DerivedSourceUtils.IndexConfigContext> indexConfigContexts
825+
) {
826+
DerivedSourceUtils.IndexConfigContext derivedSourceEnabledContext = indexConfigContexts.get(0);
827+
DerivedSourceUtils.IndexConfigContext derivedSourceDisabledContext = indexConfigContexts.get(1);
828+
DerivedSourceUtils.IndexConfigContext reindexFromEnabledToEnabledContext = indexConfigContexts.get(2);
829+
DerivedSourceUtils.IndexConfigContext reindexFromEnabledToDisabledContext = indexConfigContexts.get(3);
830+
DerivedSourceUtils.IndexConfigContext reindexFromDisabledToEnabledContext = indexConfigContexts.get(4);
831+
DerivedSourceUtils.IndexConfigContext reindexFromDisabledToDisabledContext = indexConfigContexts.get(5);
832+
833+
String originalIndexNameDerivedSourceEnabled = derivedSourceEnabledContext.indexName;
834+
String originalIndexNameDerivedSourceDisabled = derivedSourceDisabledContext.indexName;
835+
String reindexFromEnabledToEnabledIndexName = reindexFromEnabledToEnabledContext.indexName;
836+
String reindexFromEnabledToDisabledIndexName = reindexFromEnabledToDisabledContext.indexName;
837+
String reindexFromDisabledToEnabledIndexName = reindexFromDisabledToEnabledContext.indexName;
838+
String reindexFromDisabledToDisabledIndexName = reindexFromDisabledToDisabledContext.indexName;
839+
840+
createSnapshot(repository, snapshot, true);
841+
842+
deleteIndex(originalIndexNameDerivedSourceEnabled);
843+
deleteIndex(originalIndexNameDerivedSourceDisabled);
844+
deleteIndex(reindexFromEnabledToEnabledIndexName);
845+
deleteIndex(reindexFromEnabledToDisabledIndexName);
846+
deleteIndex(reindexFromDisabledToEnabledIndexName);
847+
deleteIndex(reindexFromDisabledToDisabledIndexName);
848+
849+
String restoreSuffix = "-restored";
850+
restoreSnapshot(
851+
restoreSuffix,
852+
List.of(
853+
originalIndexNameDerivedSourceEnabled,
854+
originalIndexNameDerivedSourceDisabled,
855+
reindexFromEnabledToEnabledIndexName,
856+
reindexFromEnabledToDisabledIndexName,
857+
reindexFromDisabledToEnabledIndexName,
858+
reindexFromDisabledToDisabledIndexName
859+
),
860+
repository,
861+
snapshot,
862+
true
863+
);
864+
865+
originalIndexNameDerivedSourceEnabled += restoreSuffix;
866+
originalIndexNameDerivedSourceDisabled += restoreSuffix;
867+
reindexFromEnabledToEnabledIndexName += restoreSuffix;
868+
reindexFromEnabledToDisabledIndexName += restoreSuffix;
869+
reindexFromDisabledToEnabledIndexName += restoreSuffix;
870+
reindexFromDisabledToDisabledIndexName += restoreSuffix;
871+
872+
assertIndexBigger(originalIndexNameDerivedSourceDisabled, originalIndexNameDerivedSourceEnabled);
873+
assertIndexBigger(originalIndexNameDerivedSourceDisabled, reindexFromEnabledToEnabledIndexName);
874+
assertIndexBigger(originalIndexNameDerivedSourceDisabled, reindexFromDisabledToEnabledIndexName);
875+
assertIndexBigger(reindexFromEnabledToDisabledIndexName, originalIndexNameDerivedSourceEnabled);
876+
assertIndexBigger(reindexFromDisabledToDisabledIndexName, originalIndexNameDerivedSourceEnabled);
877+
assertDocsMatch(
878+
derivedSourceDisabledContext.docCount,
879+
originalIndexNameDerivedSourceDisabled,
880+
reindexFromEnabledToEnabledIndexName
881+
);
882+
assertDocsMatch(
883+
derivedSourceDisabledContext.docCount,
884+
originalIndexNameDerivedSourceDisabled,
885+
reindexFromDisabledToEnabledIndexName
886+
);
887+
assertDocsMatch(
888+
derivedSourceDisabledContext.docCount,
889+
originalIndexNameDerivedSourceDisabled,
890+
reindexFromEnabledToDisabledIndexName
891+
);
892+
assertDocsMatch(
893+
derivedSourceDisabledContext.docCount,
894+
originalIndexNameDerivedSourceDisabled,
895+
reindexFromDisabledToDisabledIndexName
896+
);
897+
}
898+
758899
@SneakyThrows
759900
protected void assertIndexBigger(String expectedBiggerIndex, String expectedSmallerIndex) {
760901
if (isExhaustive()) {

‎src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.apache.commons.lang.StringUtils;
1717
import org.apache.hc.core5.http.io.entity.EntityUtils;
1818
import org.apache.hc.core5.net.URIBuilder;
19+
import org.hamcrest.Matchers;
1920
import org.junit.AfterClass;
2021
import org.junit.Before;
2122
import org.opensearch.Version;
@@ -24,6 +25,7 @@
2425
import org.opensearch.common.settings.Settings;
2526
import org.opensearch.common.xcontent.XContentFactory;
2627
import org.opensearch.common.xcontent.XContentHelper;
28+
import org.opensearch.common.xcontent.json.JsonXContent;
2729
import org.opensearch.core.common.bytes.BytesReference;
2830
import org.opensearch.core.rest.RestStatus;
2931
import org.opensearch.core.xcontent.DeprecationHandler;
@@ -2360,4 +2362,30 @@ protected void setupSnapshotRestore(String index, String snapshot, String reposi
23602362
createSnapshot(repository, snapshot, true);
23612363
}
23622364

2365+
protected static void restoreSnapshot(
2366+
String restoreIndexSuffix,
2367+
List<String> indices,
2368+
String repository,
2369+
String snapshot,
2370+
boolean waitForCompletion
2371+
) throws IOException {
2372+
// valid restore
2373+
XContentBuilder restoreCommand = JsonXContent.contentBuilder().startObject();
2374+
restoreCommand.field("indices", String.join(",", indices));
2375+
restoreCommand.field("rename_pattern", "(.+)");
2376+
restoreCommand.field("rename_replacement", "$1" + restoreIndexSuffix);
2377+
restoreCommand.endObject();
2378+
2379+
Request restoreRequest = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_restore");
2380+
restoreRequest.addParameter("wait_for_completion", "true");
2381+
restoreRequest.setJsonEntity(restoreCommand.toString());
2382+
2383+
final Response restoreResponse = client().performRequest(restoreRequest);
2384+
assertThat(
2385+
"Failed to restore snapshot [" + snapshot + "] from repository [" + repository + "]: " + String.valueOf(restoreResponse),
2386+
restoreResponse.getStatusLine().getStatusCode(),
2387+
Matchers.equalTo(RestStatus.OK.getStatus())
2388+
);
2389+
}
2390+
23632391
}

0 commit comments

Comments
 (0)