Skip to content

Commit f246d61

Browse files
AndreKuraitAndreKurait
authored
Merge pull request opensearch-project#1729 from AndreKurait/AutomaticFlattenedConversion
Add automatic flattened to flat_object transformation
2 parents e915c30 + 9ff5cd7 commit f246d61

File tree

7 files changed

+191
-28
lines changed

7 files changed

+191
-28
lines changed

‎MetadataMigration/src/main/java/org/opensearch/migrations/MetadataTransformationRegistry.java

Lines changed: 57 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.opensearch.migrations;
22

33
import java.util.List;
4+
import java.util.function.BiPredicate;
45
import java.util.function.Predicate;
56
import java.util.stream.Collectors;
67

@@ -35,57 +36,95 @@ public class MetadataTransformationRegistry {
3536
private static final List<TransformerConfigs> BAKED_IN_TRANSFORMER_CONFIGS = List.of(
3637
TransformerConfigs.builder()
3738
.filename("js/es-string-text-keyword-metadata.js")
38-
.isRelevantForSourceVersion(UnboundVersionMatchers.isBelowES_6_X)
39+
.isRelevantForVersions(andSourceTargetVersionPredicate(
40+
UnboundVersionMatchers.isBelowES_6_X,
41+
UnboundVersionMatchers.isBelowES_5_X.negate()
42+
))
3943
.transformerInfo(Transformers.TransformerInfo.builder()
4044
.name("Field Data Type Deprecation - string")
41-
.descriptionLine("Convert mapping type string to text/keyword based on field data mappings")
45+
.descriptionLine("Convert field data type string to text/keyword")
4246
.build())
4347
.build(),
4448
TransformerConfigs.builder()
4549
.filename("js/es-vector-knn-metadata.js")
46-
.isRelevantForSourceVersion(UnboundVersionMatchers.isGreaterOrEqualES_7_X)
50+
.isRelevantForVersions(andSourceTargetVersionPredicate(
51+
UnboundVersionMatchers.isGreaterOrEqualES_7_X,
52+
UnboundVersionMatchers.anyOS
53+
))
4754
.transformerInfo(Transformers.TransformerInfo.builder()
4855
.name("dense_vector to knn_vector")
49-
.descriptionLine("Convert mapping type dense_vector to OpenSearch knn_vector")
56+
.descriptionLine("Convert field data type dense_vector to OpenSearch knn_vector")
57+
.build())
58+
.build(),
59+
TransformerConfigs.builder()
60+
.filename("js/metadataUpdater.js")
61+
.context(
62+
"{" +
63+
" \"rules\": [" +
64+
" {" +
65+
" \"when\": { \"type\": \"flattened\" }," +
66+
" \"set\": { \"type\": \"flat_object\" }," +
67+
" \"remove\": [\"index\"]" +
68+
" }" +
69+
" ]" +
70+
"}")
71+
.isRelevantForVersions(andSourceTargetVersionPredicate(
72+
UnboundVersionMatchers.isGreaterOrEqualES_7_3,
73+
UnboundVersionMatchers.equalOrGreaterThanOS_2_7
74+
))
75+
.transformerInfo(Transformers.TransformerInfo.builder()
76+
.name("flattened to flat_object")
77+
.descriptionLine("Convert field data type flattened to OpenSearch flat_object")
5078
.build())
5179
.build()
5280
);
5381

82+
private static BiPredicate<Version, Version> andSourceTargetVersionPredicate(
83+
Predicate<Version> sourcePredicate,
84+
Predicate<Version> targetPredicate
85+
) {
86+
return (source, target) -> sourcePredicate.test(source) && targetPredicate.test(target);
87+
}
5488

5589
@Getter
5690
@Builder
5791
private static class TransformerConfigs {
5892
@NonNull private Transformers.TransformerInfo transformerInfo;
5993
@NonNull private String filename;
60-
@NonNull private Predicate<Version> isRelevantForSourceVersion;
94+
private String context;
95+
@NonNull private BiPredicate<Version, Version> isRelevantForVersions;
6196
}
6297

63-
public static Transformers getCustomTransformationBySourceVersion(Version sourceVersion) {
98+
public static Transformers getCustomTransformationByClusterVersions(Version sourceVersion, Version targetVersion) {
6499
var transformersBuilder = Transformers.builder();
65100
var bakedInTransformers = BAKED_IN_TRANSFORMER_CONFIGS
66101
.stream().filter(config ->
67-
config.getIsRelevantForSourceVersion().test(sourceVersion))
102+
config.isRelevantForVersions.test(sourceVersion, targetVersion))
68103
.toList();
69104
transformersBuilder.transformerInfos(bakedInTransformers.stream().map(TransformerConfigs::getTransformerInfo).collect(Collectors.toList()));
70-
var config = getAggregateJSTransformer(bakedInTransformers.stream().map(TransformerConfigs::getFilename).toList());
105+
var config = getAggregateJSTransformer(bakedInTransformers);
71106
logTransformerConfig("Default breaking changes transform config", config);
72107
transformersBuilder.transformer(configToTransformer(config));
73108
return transformersBuilder.build();
74109
}
75110

76-
private static String getAggregateJSTransformer(List<String> jsFilenames) {
77-
return jsFilenames.isEmpty() ? NOOP_TRANSFORMATION_CONFIG :
78-
jsFilenames.stream()
79-
.map(filename ->
80-
"{" +
81-
" \"JsonJSTransformerProvider\":{" +
82-
" \"initializationResourcePath\":\"" + filename + "\"," +
83-
" \"bindingsObject\":\"{}\"" +
84-
" }" +
85-
"}")
111+
private static String getAggregateJSTransformer(List<TransformerConfigs> transformerConfigs) {
112+
return transformerConfigs.isEmpty() ? NOOP_TRANSFORMATION_CONFIG :
113+
transformerConfigs.stream()
114+
.map(config -> getJSTransform(config.getFilename(), config.getContext()))
86115
.collect(Collectors.joining(",", "[", "]"));
87116
}
88117

118+
private static String getJSTransform(String filename, String context) {
119+
var bindings = context == null ? "{}" : context.replace("\"", "\\\"");
120+
return "{" +
121+
" \"JsonJSTransformerProvider\":{" +
122+
" \"initializationResourcePath\":\"" + filename + "\"," +
123+
" \"bindingsObject\": \"" + bindings + "\"" +
124+
" }" +
125+
"}";
126+
}
127+
89128
public static Transformer configToTransformer(String config) {
90129
var transformer = new TransformationLoader().getTransformerFactoryLoader(config);
91130
return new TransformerToIJsonTransformerAdapter(transformer);

‎MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@ protected Clusters createClusters() {
5252
return clusters.build();
5353
}
5454

55-
protected Transformers getCustomTransformer(Version sourceVersion) {
56-
var versionSpecificCustomTransforms = MetadataTransformationRegistry.getCustomTransformationBySourceVersion(sourceVersion);
55+
protected Transformers getCustomTransformer(Version sourceVersion, Version targetVersion) {
56+
var versionSpecificCustomTransforms = MetadataTransformationRegistry.getCustomTransformationByClusterVersions(sourceVersion, targetVersion);
5757
var transformerConfig = TransformerConfigUtils.getTransformerConfig(arguments.metadataCustomTransformationParams);
5858
if (transformerConfig != null) {
5959
MetadataTransformationRegistry.logTransformerConfig("User supplied custom transform", transformerConfig);
@@ -82,7 +82,7 @@ protected Transformers selectTransformer(Clusters clusters, int awarenessAttribu
8282
arguments.metadataTransformationParams,
8383
allowLooseVersionMatches
8484
);
85-
var customTransformer = getCustomTransformer(clusters.getSource().getVersion());
85+
var customTransformer = getCustomTransformer(clusters.getSource().getVersion(), clusters.getTarget().getVersion());
8686
log.atInfo().setMessage("Selected transformer composite: custom = {}, version = {}")
8787
.addArgument(customTransformer.getClass().getSimpleName())
8888
.addArgument(versionTransformer.getClass().getSimpleName())

‎MetadataMigration/src/test/java/org/opensearch/migrations/ES7FlattenedMappingsTransformationTest.java

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package org.opensearch.migrations;
2+
3+
import java.io.File;
4+
import java.util.stream.Stream;
5+
6+
import org.opensearch.migrations.bulkload.framework.SearchClusterContainer;
7+
import org.opensearch.migrations.commands.MigrationItemResult;
8+
import org.opensearch.migrations.snapshot.creation.tracing.SnapshotTestContext;
9+
10+
import lombok.SneakyThrows;
11+
import lombok.extern.slf4j.Slf4j;
12+
import org.junit.jupiter.api.Assertions;
13+
import org.junit.jupiter.api.Tag;
14+
import org.junit.jupiter.api.io.TempDir;
15+
import org.junit.jupiter.params.ParameterizedTest;
16+
import org.junit.jupiter.params.provider.Arguments;
17+
import org.junit.jupiter.params.provider.MethodSource;
18+
19+
import static org.hamcrest.CoreMatchers.containsString;
20+
import static org.hamcrest.CoreMatchers.equalTo;
21+
import static org.hamcrest.MatcherAssert.assertThat;
22+
23+
24+
@Tag("isolatedTest")
25+
@Slf4j
26+
class ES7FlattenedMappingsTransformationTest extends BaseMigrationTest {
27+
@TempDir
28+
protected File localDirectory;
29+
30+
private static Stream<Arguments> scenarios() {
31+
var source = SearchClusterContainer.ES_V7_17;
32+
var target = SearchClusterContainer.OS_LATEST;
33+
return Stream.of(Arguments.of(source, target));
34+
}
35+
36+
@ParameterizedTest(name = "Custom Transformation From {0} to {1}")
37+
@MethodSource(value = "scenarios")
38+
void customTransformationMetadataMigration(
39+
SearchClusterContainer.ContainerVersion sourceVersion,
40+
SearchClusterContainer.ContainerVersion targetVersion) {
41+
try (
42+
final var sourceCluster = new SearchClusterContainer(sourceVersion);
43+
final var targetCluster = new SearchClusterContainer(targetVersion)
44+
) {
45+
this.sourceCluster = sourceCluster;
46+
this.targetCluster = targetCluster;
47+
performCustomTransformationTest();
48+
}
49+
}
50+
51+
@SneakyThrows
52+
private void performCustomTransformationTest() {
53+
startClusters();
54+
55+
String requestBody = "{\n" +
56+
" \"mappings\": {\n" +
57+
" \"dynamic\": \"strict\",\n" +
58+
" \"properties\": {\n" +
59+
" \"flattened_data\": {\n" +
60+
" \"type\": \"flattened\"," +
61+
" \"index\": false" +
62+
" }\n" +
63+
" }\n" +
64+
" }\n" +
65+
"}";
66+
var indexName = "flattened";
67+
sourceOperations.createIndex(indexName, requestBody);
68+
69+
var snapshotName = "custom_transformation_snap";
70+
var testSnapshotContext = SnapshotTestContext.factory().noOtelTracking();
71+
createSnapshot(sourceCluster, snapshotName, testSnapshotContext);
72+
sourceCluster.copySnapshotData(localDirectory.toString());
73+
var arguments = prepareSnapshotMigrationArgs(snapshotName, localDirectory.toString());
74+
75+
// Execute migration
76+
MigrationItemResult result = executeMigration(arguments, MetadataCommands.MIGRATE);
77+
78+
// Verify the migration result
79+
log.info(result.asCliOutput());
80+
assertThat(result.getExitCode(), equalTo(0));
81+
82+
var transformation = result.getTransformations().getTransformerInfos().stream()
83+
.filter(t -> t.getName().contains("flattened"))
84+
.findAny();
85+
Assertions.assertTrue(transformation.isPresent());
86+
Assertions.assertEquals("flattened to flat_object", transformation.get().getName());
87+
88+
// Verify that the transformed index exists on the target cluster
89+
var res = targetOperations.get("/" + indexName);
90+
assertThat(res.getKey(), equalTo(200));
91+
assertThat(res.getValue(), containsString(indexName));
92+
assertThat(res.getValue(), containsString("flat_object"));
93+
94+
}
95+
}

‎MetadataMigration/src/test/java/org/opensearch/migrations/ES8VectorFieldMappingsTransformationTest.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
import lombok.SneakyThrows;
1111
import lombok.extern.slf4j.Slf4j;
12+
import org.junit.jupiter.api.Assertions;
1213
import org.junit.jupiter.api.Tag;
1314
import org.junit.jupiter.api.io.TempDir;
1415
import org.junit.jupiter.params.ParameterizedTest;
@@ -101,8 +102,11 @@ private void performCustomTransformationTest() {
101102
log.info(result.asCliOutput());
102103
assertThat(result.getExitCode(), equalTo(0));
103104

104-
assertThat(result.getTransformations().getTransformerInfos().size(), equalTo(2));
105-
assertThat(result.getTransformations().getTransformerInfos().get(0).getName(), equalTo("dense_vector to knn_vector"));
105+
var vectorTransformation = result.getTransformations().getTransformerInfos().stream()
106+
.filter(t -> t.getName().contains("dense_vector"))
107+
.findAny();
108+
Assertions.assertTrue(vectorTransformation.isPresent());
109+
Assertions.assertEquals("dense_vector to knn_vector", vectorTransformation.get().getName());
106110

107111
// Verify that the transformed index exists on the target cluster
108112
var res = targetOperations.get("/" + indexName);

‎transformation/src/main/java/org/opensearch/migrations/UnboundVersionMatchers.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,12 @@ public class UnboundVersionMatchers {
1616
public static final Predicate<Version> isBelowES_8_X = belowMajorVersion(Version.fromString("ES 8.0.0"));
1717
public static final Predicate<Version> isGreaterOrEqualES_6_X = greaterOrEqualMajorVersion(Version.fromString("ES 6.0.0"));
1818
public static final Predicate<Version> isGreaterOrEqualES_7_X = greaterOrEqualMajorVersion(Version.fromString("ES 7.0.0"));
19+
public static final Predicate<Version> isGreaterOrEqualES_7_3 = greaterOrEqualMajorVersion(Version.fromString("ES 7.3.0"));
1920
public static final Predicate<Version> isGreaterOrEqualES_7_10 = greaterOrEqualMajorVersion(Version.fromString("ES 7.10.0"));
2021
public static final Predicate<Version> anyOS = VersionMatchers.matchesFlavor(Version.fromString("OS 1.0.0"));
22+
public static final Predicate<Version> isGreaterOrEqualOS_3_x = greaterOrEqualMajorVersion(Version.fromString("OS 3.0.0"));
23+
public static final Predicate<Version> equalOrGreaterThanOS_2_7 = VersionMatchers.equalOrGreaterThanMinorVersion(Version.fromString("OS 2.7.0"))
24+
.or(isGreaterOrEqualOS_3_x);
2125

2226
static Predicate<Version> belowMajorVersion(final Version version) {
2327
return other -> {

‎transformation/standardJavascriptTransforms/src/metadataUpdater.js

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
function applyRulesToMap(when, set, remove, map) {
2-
const matches = Object.entries(when).every(([k, v]) => map.get(k) === v);
2+
const matches = [...when.entries()].every( ([k, v]) => map.get(k) === v);
33
if (matches) {
4-
Object.entries(set).every(([k, v]) => map.set(k, v));
4+
[...set.entries()].every(([k, v]) => map.set(k, v));
55
remove.forEach((key) => map.delete(key));
66
}
77
}
@@ -18,7 +18,10 @@ function applyRules(node, rules) {
1818
if (Array.isArray(node)) {
1919
node.forEach((child) => applyRules(child, rules));
2020
} else if (node instanceof Map) {
21-
for (const { when, set, remove = [] } of rules) {
21+
for (const rule of rules) {
22+
let when = rule.get("when");
23+
let set = rule.get("set");
24+
let remove = rule.get("remove") || [];
2225
applyRulesToMap(when, set, remove, node);
2326
}
2427
// recurse
@@ -35,7 +38,7 @@ function applyRules(node, rules) {
3538
}
3639

3740
function main(context) {
38-
if (!context.rules) {
41+
if (!context.rules && !context.get("rules")) {
3942
throw Error(
4043
"Expected rules to be defined in the context. Example: " +
4144
JSON.stringify(
@@ -46,6 +49,15 @@ function main(context) {
4649
);
4750
}
4851

52+
if (context instanceof Map) {
53+
return (doc) => {
54+
if (doc && doc.type && doc.name && doc.body) {
55+
applyRules(doc.body, context.get("rules"));
56+
}
57+
return doc;
58+
};
59+
}
60+
4961
return (doc) => {
5062
if (doc && doc.type && doc.name && doc.body) {
5163
applyRules(doc.body, context.rules);

‎transformation/standardJavascriptTransforms/test/metadataUpdater.test.js

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,16 @@ describe("Metadata updater internals", () => {
192192
});
193193

194194
test("Decomposes Maps", () => {
195-
const updaterInstance = main(simpleConfig);
195+
let simpleConfigMap = new Map([
196+
[
197+
"rules", Array.of(new Map( [
198+
["when", new Map([["a", 1]])],
199+
["set", new Map([["a", 2]])],
200+
["remove", Array.of("c")]
201+
]))]]
202+
);
203+
204+
const updaterInstance = main(simpleConfigMap);
196205

197206
const testObj = new Map([
198207
["a", 1],

0 commit comments

Comments
 (0)