diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 4a52444b4f20a..66ca8173bd73b 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -28,7 +28,7 @@ dependencies { restResources { restApi { - include '_common', 'ingest', 'cluster', 'indices', 'index', 'bulk', 'nodes', 'get', 'update', 'cat', 'mget', 'search', 'simulate' + include '_common', 'ingest', 'cluster', 'indices', 'index', 'bulk', 'nodes', 'get', 'update', 'cat', 'mget', 'search', 'simulate', 'capabilities' } } diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/340_flexible_access_pattern.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/340_flexible_access_pattern.yml new file mode 100644 index 0000000000000..809a3cb72dae2 --- /dev/null +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/340_flexible_access_pattern.yml @@ -0,0 +1,381 @@ +--- +setup: + - requires: + reason: "Flexible access pattern was added in 9.2+" + test_runner_features: [ capabilities ] + capabilities: + - method: PUT + path: /_ingest/pipeline/{id} + capabilities: [ 'field_access_pattern.flexible' ] + +--- +teardown: + - do: + ingest.delete_pipeline: + id: "1" + ignore: 404 + +--- +"Test dotted field name writes": + - do: + ingest.put_pipeline: + id: "1" + body: > + { + "field_access_pattern": "flexible", + "processors": [ + { + "set": { + "field": "a.b.c.d", + "value": "1" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: "no_field" + pipeline: "1" + body: { + foo: bar + } + + - do: + index: + index: test + id: "normalized" + pipeline: "1" + body: { + a: { + b: { + c: { + d: 0 + } + } + } + } + + - do: + index: + index: test + id: "dotted_only" + pipeline: "1" + body: { + a.b.c.d: 0 + } + + - do: + index: + index: test + id: "split_dots" + pipeline: "1" + body: { + a.b: { + c.d: 0 + } + } + + - do: + index: + index: test + id: "middle_dot" + pipeline: "1" + body: { + a: { + b.c: { + d: 0 + } + } + } + + - do: + get: + index: test + id: "no_field" + - match: { _source.a\.b\.c\.d: "1" } + - do: + get: + index: test + id: "normalized" + - match: { _source.a.b.c.d: "1" } + - do: + get: + index: test + id: "dotted_only" + - match: { _source.a\.b\.c\.d: "1" } + - do: + get: + index: test + id: "split_dots" + - match: { _source.a\.b.c\.d: "1" } + - do: + get: + index: test + id: "middle_dot" + - match: { _source.a.b\.c.d: "1" } + +--- +"Test dotted field name retrieval": + - do: + ingest.put_pipeline: + id: "1" + body: > + { + "field_access_pattern": "flexible", + "processors": [ + { + "set": { + "field": "result", + "copy_from": "a.b.c.d" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: "normalized" + pipeline: "1" + body: { + a: { + b: { + c: { + d: 0 + } + } + } + } + + - do: + index: + index: test + id: "dotted_only" + pipeline: "1" + body: { + a.b.c.d: 0 + } + + - do: + index: + index: test + id: "split_dots" + pipeline: "1" + body: { + a.b: { + c.d: 0 + } + } + + - do: + index: + index: test + id: "middle_dot" + pipeline: "1" + body: { + a: { + b.c: { + d: 0 + } + } + } + + - do: + get: + index: test + id: "normalized" + - match: { _source.result: 0 } + - do: + get: + index: test + id: "dotted_only" + - match: { _source.result: 0 } + - do: + get: + index: test + id: "split_dots" + - match: { _source.result: 0 } + - do: + get: + index: test + id: "middle_dot" + - match: { _source.result: 0 } + +--- +"Test dotted field name exists": + - do: + ingest.put_pipeline: + id: "1" + body: > + { + "field_access_pattern": "flexible", + "processors": [ + { + "rename": { + "field": "foo", + "target_field": "a.b.c.d", + "override": false + } + } + ] + } + - match: { acknowledged: true } + + - do: + catch: bad_request + index: + index: test + id: "normalized" + pipeline: "1" + body: { + foo: "bar", + a: { + b: { + c: { + d: 0 + } + } + } + } + - match: { error.root_cause.0.reason: "field [a.b.c.d] already exists" } + + - do: + catch: bad_request + index: + index: test + id: "dotted_only" + pipeline: "1" + body: { + foo: "bar", + a.b.c.d: 0 + } + - match: { error.root_cause.0.reason: "field [a.b.c.d] already exists" } + + - do: + catch: bad_request + index: + index: test + id: "split_dots" + pipeline: "1" + body: { + foo: "bar", + a.b: { + c.d: 0 + } + } + - match: { error.root_cause.0.reason: "field [a.b.c.d] already exists" } + + - do: + catch: bad_request + index: + index: test + id: "middle_dot" + pipeline: "1" + body: { + foo: "bar", + a: { + b.c: { + d: 0 + } + } + } + - match: { error.root_cause.0.reason: "field [a.b.c.d] already exists" } + +--- +"Test dotted field removal": + - do: + ingest.put_pipeline: + id: "1" + body: > + { + "field_access_pattern": "flexible", + "processors": [ + { + "remove": { + "field": "a.b.c.d" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: "normalized" + pipeline: "1" + body: { + foo: "bar", + a: { + b: { + c: { + d: 0 + } + } + } + } + + - do: + index: + index: test + id: "dotted_only" + pipeline: "1" + body: { + foo: "bar", + a.b.c.d: 0 + } + + - do: + index: + index: test + id: "split_dots" + pipeline: "1" + body: { + foo: "bar", + a.b: { + c.d: 0 + } + } + + - do: + index: + index: test + id: "middle_dot" + pipeline: "1" + body: { + foo: "bar", + a: { + b.c: { + d: 0 + } + } + } + + - do: + get: + index: test + id: "normalized" + - match: { _source.foo: "bar" } + - is_false: _source.a.b.c.d + - do: + get: + index: test + id: "dotted_only" + - match: { _source.foo: "bar" } + - is_false: _source.a\.b\.c\.d + - do: + get: + index: test + id: "split_dots" + - match: { _source.foo: "bar" } + - is_false: _source.a\.b.c\.d + - do: + get: + index: test + id: "middle_dot" + - match: { _source.foo: "bar" } + - is_false: _source.a.b\.c.d diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index af3bf3eb21f98..a9a738bfdb806 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -203,13 +203,16 @@ public T getFieldValue(String path, Class clazz) { public T getFieldValue(String path, Class clazz, boolean ignoreMissing) { final FieldPath fieldPath = FieldPath.of(path); Object context = fieldPath.initialContext(this); - ResolveResult result = resolve(fieldPath.pathElements, fieldPath.pathElements.length, path, context); + ResolveResult result = resolve(fieldPath.pathElements, fieldPath.pathElements.length, path, context, getCurrentAccessPatternSafe()); if (result.wasSuccessful) { return cast(path, result.resolvedObject, clazz); } else if (ignoreMissing) { return null; } else { - throw new IllegalArgumentException(result.errorMessage); + // Reconstruct the error message if the resolve result was incomplete + throw new IllegalArgumentException( + Objects.requireNonNullElseGet(result.errorMessage, () -> Errors.notPresent(path, result.missingFields)) + ); } } @@ -269,15 +272,89 @@ public boolean hasField(String path) { public boolean hasField(String path, boolean failOutOfRange) { final FieldPath fieldPath = FieldPath.of(path); Object context = fieldPath.initialContext(this); - for (int i = 0; i < fieldPath.pathElements.length - 1; i++) { + int leafKeyIndex = fieldPath.pathElements.length - 1; + int lastContainerIndex = fieldPath.pathElements.length - 2; + String leafKey = fieldPath.pathElements[leafKeyIndex]; + for (int i = 0; i <= lastContainerIndex; i++) { String pathElement = fieldPath.pathElements[i]; if (context == null) { return false; } else if (context instanceof IngestCtxMap map) { // optimization: handle IngestCtxMap separately from Map - context = map.get(pathElement); + switch (getCurrentAccessPatternSafe()) { + case CLASSIC -> context = map.get(pathElement); + case FLEXIBLE -> { + Object object = map.getOrDefault(pathElement, NOT_FOUND); + if (object != NOT_FOUND) { + context = object; + } else if (i == lastContainerIndex) { + // This is the last path element, update the leaf key to use this path element as a dotted prefix. + // Leave the context as it is. + leafKey = pathElement + "." + leafKey; + } else { + // Iterate through the remaining path elements, joining them with dots, until we get a hit + String combinedPath = pathElement; + for (int j = i + 1; j <= lastContainerIndex; j++) { + combinedPath = combinedPath + "." + fieldPath.pathElements[j]; + object = map.getOrDefault(combinedPath, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object != NOT_FOUND) { + // Found one, update the outer loop index to skip past the elements we've used + context = object; + i = j; + break; + } + } + if (object == NOT_FOUND) { + // Made it to the last path element without finding the field. + // Update the leaf key to use the visited combined path elements as a dotted prefix. + leafKey = combinedPath + "." + leafKey; + // Update outer loop index to skip past the elements we've used + i = lastContainerIndex; + } + } + } + } } else if (context instanceof Map map) { - context = map.get(pathElement); + switch (getCurrentAccessPatternSafe()) { + case CLASSIC -> context = map.get(pathElement); + case FLEXIBLE -> { + @SuppressWarnings("unchecked") + Map typedMap = (Map) context; + Object object = typedMap.getOrDefault(pathElement, NOT_FOUND); + if (object != NOT_FOUND) { + context = object; + } else if (i == lastContainerIndex) { + // This is the last path element, update the leaf key to use this path element as a dotted prefix. + // Leave the context as it is. + leafKey = pathElement + "." + leafKey; + } else { + // Iterate through the remaining path elements, joining them with dots, until we get a hit + String combinedPath = pathElement; + for (int j = i + 1; j <= lastContainerIndex; j++) { + combinedPath = combinedPath + "." + fieldPath.pathElements[j]; + object = typedMap.getOrDefault(combinedPath, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object != NOT_FOUND) { + // Found one, update the outer loop index to skip past the elements we've used + context = object; + i = j; + break; + } + } + if (object == NOT_FOUND) { + // Made it to the last path element without finding the field. + // Update the leaf key to use the visited combined path elements as a dotted prefix. + leafKey = combinedPath + "." + leafKey; + // Update outer loop index to skip past the elements we've used. + i = lastContainerIndex; + } + } + } + } } else if (context instanceof List list) { + if (getCurrentAccessPatternSafe() == IngestPipelineFieldAccessPattern.FLEXIBLE) { + // Flexible access pattern cannot yet access array values, new syntax must be added. + // Handle this as if the path element was not parsable as an integer in the classic mode + return false; + } int index; try { index = Integer.parseInt(pathElement); @@ -298,7 +375,6 @@ public boolean hasField(String path, boolean failOutOfRange) { } } - String leafKey = fieldPath.pathElements[fieldPath.pathElements.length - 1]; if (context == null) { return false; } else if (context instanceof IngestCtxMap map) { // optimization: handle IngestCtxMap separately from Map @@ -306,6 +382,11 @@ public boolean hasField(String path, boolean failOutOfRange) { } else if (context instanceof Map map) { return map.containsKey(leafKey); } else if (context instanceof List list) { + if (getCurrentAccessPatternSafe() == IngestPipelineFieldAccessPattern.FLEXIBLE) { + // Flexible access pattern cannot yet access array values, new syntax must be added. + // Handle this as if the path element was not parsable as an integer in the classic mode + return false; + } try { int index = Integer.parseInt(leafKey); if (index >= 0 && index < list.size()) { @@ -345,16 +426,26 @@ public void removeField(String path) { public void removeField(String path, boolean ignoreMissing) { final FieldPath fieldPath = FieldPath.of(path); Object context = fieldPath.initialContext(this); - ResolveResult result = resolve(fieldPath.pathElements, fieldPath.pathElements.length - 1, path, context); + String leafKey = fieldPath.pathElements[fieldPath.pathElements.length - 1]; + ResolveResult result = resolve( + fieldPath.pathElements, + fieldPath.pathElements.length - 1, + path, + context, + getCurrentAccessPatternSafe() + ); if (result.wasSuccessful) { context = result.resolvedObject; + } else if (result.missingFields != null) { + // Incomplete result, update the leaf key and context to continue the operation + leafKey = result.missingFields + "." + leafKey; + context = result.resolvedObject; } else if (ignoreMissing) { return; // nothing was found, so there's nothing to remove :shrug: } else { throw new IllegalArgumentException(result.errorMessage); } - String leafKey = fieldPath.pathElements[fieldPath.pathElements.length - 1]; if (context == null && ignoreMissing == false) { throw new IllegalArgumentException(Errors.cannotRemove(path, leafKey, null)); } else if (context instanceof IngestCtxMap map) { // optimization: handle IngestCtxMap separately from Map @@ -370,6 +461,15 @@ public void removeField(String path, boolean ignoreMissing) { throw new IllegalArgumentException(Errors.notPresent(path, leafKey)); } } else if (context instanceof List list) { + if (getCurrentAccessPatternSafe() == IngestPipelineFieldAccessPattern.FLEXIBLE) { + // Flexible access pattern cannot yet access array values, new syntax must be added. + if (ignoreMissing == false) { + throw new IllegalArgumentException("path [" + path + "] is not valid"); + } else { + // ignoreMissing is true, so treat this as if we had just not found the field. + return; + } + } int index = -1; try { index = Integer.parseInt(leafKey); @@ -394,28 +494,102 @@ public void removeField(String path, boolean ignoreMissing) { * Resolves the path elements (up to the limit) within the context. The result of such resolution can either be successful, * or can indicate a failure. */ - private static ResolveResult resolve(final String[] pathElements, final int limit, final String fullPath, Object context) { + private static ResolveResult resolve( + final String[] pathElements, + final int limit, + final String fullPath, + Object context, + IngestPipelineFieldAccessPattern accessPattern + ) { for (int i = 0; i < limit; i++) { String pathElement = pathElements[i]; if (context == null) { return ResolveResult.error(Errors.cannotResolve(fullPath, pathElement, null)); } else if (context instanceof IngestCtxMap map) { // optimization: handle IngestCtxMap separately from Map - Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get - if (object == NOT_FOUND) { - return ResolveResult.error(Errors.notPresent(fullPath, pathElement)); - } else { - context = object; + switch (accessPattern) { + case CLASSIC -> { + Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object == NOT_FOUND) { + return ResolveResult.error(Errors.notPresent(fullPath, pathElement)); + } else { + context = object; + } + } + case FLEXIBLE -> { + Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object != NOT_FOUND) { + context = object; + } else if (i == (limit - 1)) { + // This is our last path element, return incomplete + return ResolveResult.incomplete(context, pathElement); + } else { + // Attempt a flexible lookup + // Iterate through the remaining elements until we get a hit + String combinedPath = pathElement; + for (int j = i + 1; j < limit; j++) { + combinedPath = combinedPath + "." + pathElements[j]; + object = map.getOrDefault(combinedPath, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object != NOT_FOUND) { + // Found one, update the outer loop index to skip past the elements we've used + context = object; + i = j; + break; + } + } + if (object == NOT_FOUND) { + // Not found, and out of path elements, return an incomplete result + return ResolveResult.incomplete(context, combinedPath); + } + } + } } } else if (context instanceof Map) { - @SuppressWarnings("unchecked") - Map map = (Map) context; - Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get - if (object == NOT_FOUND) { - return ResolveResult.error(Errors.notPresent(fullPath, pathElement)); - } else { - context = object; + switch (accessPattern) { + case CLASSIC -> { + @SuppressWarnings("unchecked") + Map map = (Map) context; + Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object == NOT_FOUND) { + return ResolveResult.error(Errors.notPresent(fullPath, pathElement)); + } else { + context = object; + } + } + case FLEXIBLE -> { + @SuppressWarnings("unchecked") + Map map = (Map) context; + Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object != NOT_FOUND) { + context = object; + } else if (i == (limit - 1)) { + // This is our last path element, return incomplete + return ResolveResult.incomplete(context, pathElement); + } else { + // Attempt a flexible lookup + // Iterate through the remaining elements until we get a hit + String combinedPath = pathElement; + for (int j = i + 1; j < limit; j++) { + combinedPath = combinedPath + "." + pathElements[j]; + object = map.getOrDefault(combinedPath, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object != NOT_FOUND) { + // Found one, update the outer loop index to skip past the elements we've used + context = object; + i = j; + break; + } + } + if (object == NOT_FOUND) { + // Not found, and out of path elements, return an incomplete result + return ResolveResult.incomplete(context, combinedPath); + } + } + } } } else if (context instanceof List list) { + if (accessPattern == IngestPipelineFieldAccessPattern.FLEXIBLE) { + // Flexible access pattern cannot yet access array values, new syntax must be added. + return ResolveResult.error(Errors.invalidPath(fullPath)); + } int index; try { index = Integer.parseInt(pathElement); @@ -562,31 +736,108 @@ public void setFieldValue(String path, Object value, boolean ignoreEmptyValue) { private void setFieldValue(String path, Object value, boolean append, boolean allowDuplicates) { final FieldPath fieldPath = FieldPath.of(path); Object context = fieldPath.initialContext(this); - for (int i = 0; i < fieldPath.pathElements.length - 1; i++) { + int leafKeyIndex = fieldPath.pathElements.length - 1; + int lastContainerIndex = fieldPath.pathElements.length - 2; + String leafKey = fieldPath.pathElements[leafKeyIndex]; + for (int i = 0; i <= lastContainerIndex; i++) { String pathElement = fieldPath.pathElements[i]; if (context == null) { throw new IllegalArgumentException(Errors.cannotResolve(path, pathElement, null)); } else if (context instanceof IngestCtxMap map) { // optimization: handle IngestCtxMap separately from Map - Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get - if (object == NOT_FOUND) { - Map newMap = new HashMap<>(); - map.put(pathElement, newMap); - context = newMap; - } else { - context = object; + switch (getCurrentAccessPatternSafe()) { + case CLASSIC -> { + Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object == NOT_FOUND) { + Map newMap = new HashMap<>(); + map.put(pathElement, newMap); + context = newMap; + } else { + context = object; + } + } + case FLEXIBLE -> { + Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object != NOT_FOUND) { + context = object; + } else if (i == lastContainerIndex) { + // This is our last path element, update the leaf key to use this path element as a dotted prefix. + // Leave the context as it is. + leafKey = pathElement + "." + leafKey; + } else { + // Iterate through the remaining path elements, joining them with dots, until we get a hit + String combinedPath = pathElement; + for (int j = i + 1; j <= lastContainerIndex; j++) { + combinedPath = combinedPath + "." + fieldPath.pathElements[j]; + object = map.getOrDefault(combinedPath, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object != NOT_FOUND) { + // Found one, update the outer loop index to skip past the elements we've used + context = object; + i = j; + break; + } + } + if (object == NOT_FOUND) { + // Made it to the last path element without finding the field. + // Update the leaf key to use the visited combined path elements as a dotted prefix. + leafKey = combinedPath + "." + leafKey; + // Update outer loop index to skip past the elements we've used + i = lastContainerIndex; + } + } + } } } else if (context instanceof Map) { - @SuppressWarnings("unchecked") - Map map = (Map) context; - Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get - if (object == NOT_FOUND) { - Map newMap = new HashMap<>(); - map.put(pathElement, newMap); - context = newMap; - } else { - context = object; + switch (getCurrentAccessPatternSafe()) { + case CLASSIC -> { + @SuppressWarnings("unchecked") + Map map = (Map) context; + Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object == NOT_FOUND) { + Map newMap = new HashMap<>(); + map.put(pathElement, newMap); + context = newMap; + } else { + context = object; + } + } + case FLEXIBLE -> { + @SuppressWarnings("unchecked") + Map map = (Map) context; + Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object != NOT_FOUND) { + context = object; + } else if (i == lastContainerIndex) { + // This is our last path element, update the leaf key to use this path element as a dotted prefix. + // Leave the context as it is. + leafKey = pathElement + "." + leafKey; + } else { + // Iterate through the remaining path elements, joining them with dots, until we get a hit + String combinedPath = pathElement; + for (int j = i + 1; j <= lastContainerIndex; j++) { + combinedPath = combinedPath + "." + fieldPath.pathElements[j]; + object = map.getOrDefault(combinedPath, NOT_FOUND); // getOrDefault is faster than containsKey + get + if (object != NOT_FOUND) { + // Found one, update the outer loop index to skip past the elements we've used + context = object; + i = j; + break; + } + } + if (object == NOT_FOUND) { + // Made it to the last path element without finding the field. + // Update the leaf key to use the visited combined path elements as a dotted prefix. + leafKey = combinedPath + "." + leafKey; + // Update outer loop index to skip past the elements we've used + i = lastContainerIndex; + } + } + } } } else if (context instanceof List list) { + if (getCurrentAccessPatternSafe() == IngestPipelineFieldAccessPattern.FLEXIBLE) { + // Flexible access pattern cannot yet access array values, new syntax must be added. + throw new IllegalArgumentException("path [" + path + "] is not valid"); + } int index; try { index = Integer.parseInt(pathElement); @@ -603,7 +854,6 @@ private void setFieldValue(String path, Object value, boolean append, boolean al } } - String leafKey = fieldPath.pathElements[fieldPath.pathElements.length - 1]; if (context == null) { throw new IllegalArgumentException(Errors.cannotSet(path, leafKey, null)); } else if (context instanceof IngestCtxMap map) { // optimization: handle IngestCtxMap separately from Map @@ -641,6 +891,10 @@ private void setFieldValue(String path, Object value, boolean append, boolean al } map.put(leafKey, value); } else if (context instanceof List) { + if (getCurrentAccessPatternSafe() == IngestPipelineFieldAccessPattern.FLEXIBLE) { + // Flexible access pattern cannot yet access array values, new syntax must be added. + throw new IllegalArgumentException("path [" + path + "] is not valid"); + } @SuppressWarnings("unchecked") List list = (List) context; int index; @@ -905,6 +1159,14 @@ public IngestPipelineFieldAccessPattern getCurrentAccessPattern() { return accessPatternStack.peek(); } + /** + * @return The access pattern for any currently executing pipelines, or {@link IngestPipelineFieldAccessPattern#CLASSIC} if no + * pipelines are in progress for this doc for the sake of backwards compatibility + */ + private IngestPipelineFieldAccessPattern getCurrentAccessPatternSafe() { + return Objects.requireNonNullElse(getCurrentAccessPattern(), IngestPipelineFieldAccessPattern.CLASSIC); + } + /** * Adds an index to the index history for this document, returning true if the index * was added to the index history (i.e. if it wasn't already in the index history). @@ -1080,13 +1342,36 @@ public Object initialContext(IngestDocument document) { } } - private record ResolveResult(boolean wasSuccessful, Object resolvedObject, String errorMessage) { + private record ResolveResult(boolean wasSuccessful, Object resolvedObject, String errorMessage, String missingFields) { + /** + * The resolve operation ended with a successful result, locating the resolved object at the given path location + * @param resolvedObject The resolved object + * @return Successful result + */ static ResolveResult success(Object resolvedObject) { - return new ResolveResult(true, resolvedObject, null); + return new ResolveResult(true, resolvedObject, null, null); } + /** + * Due to the access pattern, the resolve operation was only partially completed. The last resolved context object is returned, + * along with the fields that have been tried up until running into the field limit. The result's success flag is set to false, + * but it contains additional information about further resolving the operation. + * @param lastResolvedObject The last successfully resolved context object from the document + * @param missingFields The fields from the given path that have not been located yet + * @return Incomplete result + */ + static ResolveResult incomplete(Object lastResolvedObject, String missingFields) { + return new ResolveResult(false, lastResolvedObject, null, missingFields); + } + + /** + * The resolve operation ended with an error. The object at the given path location could not be resolved, either due to it + * being missing, or the path being invalid. + * @param errorMessage The error message to be returned. + * @return Error result + */ static ResolveResult error(String errorMessage) { - return new ResolveResult(false, null, errorMessage); + return new ResolveResult(false, null, errorMessage, null); } } @@ -1219,5 +1504,9 @@ private static String notPresent(String path, String key) { private static String notStringOrByteArray(String path, Object value) { return "Content field [" + path + "] of unknown type [" + value.getClass().getName() + "], must be string or byte array"; } + + private static String invalidPath(String fullPath) { + return "path [" + fullPath + "] is not valid"; + } } } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index ea8e9d3843296..250ab60f7442c 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -1562,7 +1562,7 @@ synchronized void innerUpdatePipelines(ProjectId projectId, IngestMetadata newIn processorFactories, scriptService, projectId, - (nodeFeature) -> featureService.clusterHasFeature(clusterService.state(), nodeFeature) + (nodeFeature) -> featureService.clusterHasFeature(state, nodeFeature) ); newPipelines.put(newConfiguration.getId(), new PipelineHolder(newConfiguration, newPipeline)); diff --git a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java index bb1ba0d4309ad..90e50842ef553 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.ingest.PutPipelineTransportAction; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.bytes.ReleasableBytesReference; import org.elasticsearch.core.Tuple; import org.elasticsearch.rest.BaseRestHandler; @@ -78,6 +79,10 @@ public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient cl @Override public Set supportedCapabilities() { // pipeline_tracking info: `{created,modified}_date` system properties defined within pipeline definition. - return Set.of("pipeline_tracking_info"); + if (DataStream.LOGS_STREAM_FEATURE_FLAG) { + return Set.of("pipeline_tracking_info", "field_access_pattern.flexible"); + } else { + return Set.of("pipeline_tracking_info"); + } } } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java index f3825ca7f9cc1..2e308b91c58df 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java @@ -28,10 +28,14 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; +import java.util.function.Consumer; import java.util.stream.DoubleStream; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; +import static org.elasticsearch.ingest.IngestPipelineFieldAccessPattern.CLASSIC; +import static org.elasticsearch.ingest.IngestPipelineFieldAccessPattern.FLEXIBLE; import static org.hamcrest.Matchers.both; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -96,115 +100,250 @@ public void setTestIngestDocument() { DoubleStream.generate(ESTestCase::randomDouble).limit(randomInt(1000)).toArray() } ); + var dots = new HashMap<>( + Map.of( + "foo.bar.baz", + "fizzbuzz", + "dotted.integers", + new HashMap<>(Map.of("a", 1, "b.c", 2, "d.e.f", 3, "g.h.i.j", 4, "k.l.m.n.o", 5)), + "inaccessible", + new HashMap<>(Map.of("a", new HashMap<>(Map.of("b", new HashMap<>(Map.of("c", "visible")))), "a.b.c", "inaccessible")), + "arrays", + new HashMap<>( + Map.of( + "dotted.strings", + new ArrayList<>(List.of("a", "b", "c", "d")), + "dotted.objects", + new ArrayList<>(List.of(new HashMap<>(Map.of("foo", "bar")), new HashMap<>(Map.of("baz", "qux")))), + "dotted.other", + new ArrayList<>() { + { + add(null); + add(""); + } + } + ) + ), + "single_fieldname", + new HashMap<>( + Map.of( + "multiple.fieldnames", + new HashMap<>(Map.of("single_fieldname_again", new HashMap<>(Map.of("multiple.fieldnames.again", "result")))) + ) + ) + ) + ); + dots.put("foo.bar.null", null); + document.put("dots", dots); + document.put("dotted.bar.baz", true); + document.put("dotted.foo.bar.baz", new HashMap<>(Map.of("qux.quux", true))); + document.put("dotted.bar.baz_null", null); + this.document = new IngestDocument("index", "id", 1, null, null, document); } - public void testSimpleGetFieldValue() { - assertThat(document.getFieldValue("foo", String.class), equalTo("bar")); - assertThat(document.getFieldValue("int", Integer.class), equalTo(123)); - assertThat(document.getFieldValue("_source.foo", String.class), equalTo("bar")); - assertThat(document.getFieldValue("_source.int", Integer.class), equalTo(123)); - assertThat(document.getFieldValue("_index", String.class), equalTo("index")); - assertThat(document.getFieldValue("_id", String.class), equalTo("id")); - assertThat( - document.getFieldValue("_ingest.timestamp", ZonedDateTime.class), - both(notNullValue()).and(not(equalTo(BOGUS_TIMESTAMP))) + /** + * Executes an action against an ingest document using the provided access pattern. A synthetic pipeline instance with the provided + * access pattern is created and executed against the ingest document, thus updating its internal access pattern. + * @param accessPattern The access pattern to use when executing the block of code + * @param action A consumer which takes the updated ingest document and performs an action with it + * @throws Exception Any exception thrown from the provided consumer + */ + private void doWithAccessPattern(IngestPipelineFieldAccessPattern accessPattern, Consumer action) throws Exception { + AtomicReference exceptionAtomicReference = new AtomicReference<>(null); + document.executePipeline( + new Pipeline( + randomAlphanumericOfLength(10), + null, + null, + null, + new CompoundProcessor(new TestProcessor(action)), + accessPattern, + null, + null, + null + ), + (ignored, ex) -> { + if (ex != null) { + if (ex instanceof IngestProcessorException ingestProcessorException) { + exceptionAtomicReference.set((Exception) ingestProcessorException.getCause()); + } else { + exceptionAtomicReference.set(ex); + } + } + } ); - assertThat(document.getFieldValue("_source._ingest.timestamp", ZonedDateTime.class), equalTo(BOGUS_TIMESTAMP)); + Exception exception = exceptionAtomicReference.get(); + if (exception != null) { + throw exception; + } } - public void testGetFieldValueIgnoreMissing() { - assertThat(document.getFieldValue("foo", String.class, randomBoolean()), equalTo("bar")); - assertThat(document.getFieldValue("int", Integer.class, randomBoolean()), equalTo(123)); + /** + * Executes an action against an ingest document using a randomly selected access pattern. A synthetic pipeline instance with the + * selected access pattern is created and executed against the ingest document, thus updating its internal access pattern. + * @param action A consumer which takes the updated ingest document and performs an action with it + * @throws Exception Any exception thrown from the provided consumer + */ + private void doWithRandomAccessPattern(Consumer action) throws Exception { + doWithAccessPattern(randomFrom(IngestPipelineFieldAccessPattern.values()), action); + } + + public void testSimpleGetFieldValue() throws Exception { + doWithRandomAccessPattern((doc) -> { + assertThat(doc.getFieldValue("foo", String.class), equalTo("bar")); + assertThat(doc.getFieldValue("int", Integer.class), equalTo(123)); + assertThat(doc.getFieldValue("_source.foo", String.class), equalTo("bar")); + assertThat(doc.getFieldValue("_source.int", Integer.class), equalTo(123)); + assertThat(doc.getFieldValue("_index", String.class), equalTo("index")); + assertThat(doc.getFieldValue("_id", String.class), equalTo("id")); + assertThat( + doc.getFieldValue("_ingest.timestamp", ZonedDateTime.class), + both(notNullValue()).and(not(equalTo(BOGUS_TIMESTAMP))) + ); + assertThat(doc.getFieldValue("_source._ingest.timestamp", ZonedDateTime.class), equalTo(BOGUS_TIMESTAMP)); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + assertThat(doc.getFieldValue("dots.foo.bar.baz", String.class), equalTo("fizzbuzz")); + assertThat(doc.getFieldValue("dotted.bar.baz", Boolean.class), equalTo(true)); + }); + } - // if ignoreMissing is true, we just return nulls for values that aren't found - assertThat(document.getFieldValue("nonsense", Integer.class, true), nullValue()); - assertThat(document.getFieldValue("some.nonsense", Integer.class, true), nullValue()); - assertThat(document.getFieldValue("fizz.some.nonsense", Integer.class, true), nullValue()); + public void testGetFieldValueIgnoreMissing() throws Exception { + doWithRandomAccessPattern((doc) -> { + assertThat(doc.getFieldValue("foo", String.class, randomBoolean()), equalTo("bar")); + assertThat(doc.getFieldValue("int", Integer.class, randomBoolean()), equalTo(123)); - // if ignoreMissing is false, we throw an exception for values that aren't found - IllegalArgumentException e; - e = expectThrows(IllegalArgumentException.class, () -> document.getFieldValue("fizz.some.nonsense", Integer.class, false)); - assertThat(e.getMessage(), is("field [some] not present as part of path [fizz.some.nonsense]")); + // if ignoreMissing is true, we just return nulls for values that aren't found + assertThat(doc.getFieldValue("nonsense", Integer.class, true), nullValue()); + assertThat(doc.getFieldValue("some.nonsense", Integer.class, true), nullValue()); + assertThat(doc.getFieldValue("fizz.some.nonsense", Integer.class, true), nullValue()); + }); + doWithAccessPattern(CLASSIC, (doc) -> { + // if ignoreMissing is false, we throw an exception for values that aren't found + IllegalArgumentException e; + e = expectThrows(IllegalArgumentException.class, () -> doc.getFieldValue("fizz.some.nonsense", Integer.class, false)); + assertThat(e.getMessage(), is("field [some] not present as part of path [fizz.some.nonsense]")); + + // if ignoreMissing is true, and the object is present-and-of-the-wrong-type, then we also throw an exception + e = expectThrows(IllegalArgumentException.class, () -> doc.getFieldValue("int", Boolean.class, true)); + assertThat(e.getMessage(), is("field [int] of type [java.lang.Integer] cannot be cast to [java.lang.Boolean]")); + }); + doWithAccessPattern(FLEXIBLE, (doc -> { + // if ignoreMissing is false, we throw an exception for values that aren't found + IllegalArgumentException e; + e = expectThrows(IllegalArgumentException.class, () -> doc.getFieldValue("fizz.some.nonsense", Integer.class, false)); + assertThat(e.getMessage(), is("field [some.nonsense] not present as part of path [fizz.some.nonsense]")); - // if ignoreMissing is true, and the object is present-and-of-the-wrong-type, then we also throw an exception - e = expectThrows(IllegalArgumentException.class, () -> document.getFieldValue("int", Boolean.class, true)); - assertThat(e.getMessage(), is("field [int] of type [java.lang.Integer] cannot be cast to [java.lang.Boolean]")); + // if ignoreMissing is true, and the object is present-and-of-the-wrong-type, then we also throw an exception + e = expectThrows(IllegalArgumentException.class, () -> doc.getFieldValue("int", Boolean.class, true)); + assertThat(e.getMessage(), is("field [int] of type [java.lang.Integer] cannot be cast to [java.lang.Boolean]")); + })); } - public void testGetSourceObject() { + public void testGetSourceObject() throws Exception { try { - document.getFieldValue("_source", Object.class); + doWithRandomAccessPattern((doc) -> doc.getFieldValue("_source", Object.class)); fail("get field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("field [_source] not present as part of path [_source]")); } } - public void testGetIngestObject() { - assertThat(document.getFieldValue("_ingest", Map.class), notNullValue()); + public void testGetIngestObject() throws Exception { + doWithRandomAccessPattern((doc) -> assertThat(doc.getFieldValue("_ingest", Map.class), notNullValue())); } - public void testGetEmptyPathAfterStrippingOutPrefix() { + public void testGetEmptyPathAfterStrippingOutPrefix() throws Exception { try { - document.getFieldValue("_source.", Object.class); + doWithRandomAccessPattern((doc) -> doc.getFieldValue("_source.", Object.class)); fail("get field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path [_source.] is not valid")); } try { - document.getFieldValue("_ingest.", Object.class); + doWithRandomAccessPattern((doc) -> doc.getFieldValue("_ingest.", Object.class)); fail("get field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path [_ingest.] is not valid")); } } - public void testGetFieldValueNullValue() { - assertThat(document.getFieldValue("fizz.foo_null", Object.class), nullValue()); + public void testGetFieldValueNullValue() throws Exception { + doWithRandomAccessPattern((doc) -> assertThat(doc.getFieldValue("fizz.foo_null", Object.class), nullValue())); } - public void testSimpleGetFieldValueTypeMismatch() { + public void testSimpleGetFieldValueTypeMismatch() throws Exception { try { - document.getFieldValue("int", String.class); + doWithRandomAccessPattern((doc) -> doc.getFieldValue("int", String.class)); fail("getFieldValue should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("field [int] of type [java.lang.Integer] cannot be cast to [java.lang.String]")); } try { - document.getFieldValue("foo", Integer.class); + doWithRandomAccessPattern((doc) -> doc.getFieldValue("foo", Integer.class)); fail("getFieldValue should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("field [foo] of type [java.lang.String] cannot be cast to [java.lang.Integer]")); } } - public void testSimpleGetFieldValueIgnoreMissingAndTypeMismatch() { + public void testSimpleGetFieldValueIgnoreMissingAndTypeMismatch() throws Exception { try { - document.getFieldValue("int", String.class, randomBoolean()); + doWithRandomAccessPattern((doc) -> doc.getFieldValue("int", String.class, randomBoolean())); fail("getFieldValue should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("field [int] of type [java.lang.Integer] cannot be cast to [java.lang.String]")); } try { - document.getFieldValue("foo", Integer.class, randomBoolean()); + doWithRandomAccessPattern((doc) -> doc.getFieldValue("foo", Integer.class, randomBoolean())); fail("getFieldValue should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("field [foo] of type [java.lang.String] cannot be cast to [java.lang.Integer]")); } } - public void testNestedGetFieldValue() { - assertThat(document.getFieldValue("fizz.buzz", String.class), equalTo("hello world")); - assertThat(document.getFieldValue("fizz.1", String.class), equalTo("bar")); + public void testNestedGetFieldValue() throws Exception { + doWithRandomAccessPattern((doc) -> { + assertThat(doc.getFieldValue("fizz.buzz", String.class), equalTo("hello world")); + assertThat(doc.getFieldValue("fizz.1", String.class), equalTo("bar")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + // Several layers of dotted field names dots -> dotted.integers -> [a - k.l.m.n.o] + assertThat(doc.getFieldValue("dots.dotted.integers.a", Integer.class), equalTo(1)); + assertThat(doc.getFieldValue("dots.dotted.integers.b.c", Integer.class), equalTo(2)); + assertThat(doc.getFieldValue("dots.dotted.integers.d.e.f", Integer.class), equalTo(3)); + assertThat(doc.getFieldValue("dots.dotted.integers.g.h.i.j", Integer.class), equalTo(4)); + assertThat(doc.getFieldValue("dots.dotted.integers.k.l.m.n.o", Integer.class), equalTo(5)); + + // The dotted field {dots: {inaccessible: {a.b.c: "inaccessible"}}} is inaccessible because + // the field {dots: {inaccessible: {a: {b: {c: "visible"}}}}} exists + assertThat(doc.getFieldValue("dots.inaccessible.a.b.c", String.class), equalTo("visible")); + + // Mixing multiple single tokens with dotted tokens + assertThat( + doc.getFieldValue( + "dots.single_fieldname.multiple.fieldnames.single_fieldname_again.multiple.fieldnames.again", + String.class + ), + equalTo("result") + ); + + // Flexible can retrieve list objects + assertThat(doc.getFieldValue("dots.arrays.dotted.strings", List.class), equalTo(new ArrayList<>(List.of("a", "b", "c", "d")))); + assertThat( + doc.getFieldValue("dots.arrays.dotted.objects", List.class), + equalTo(new ArrayList<>(List.of(new HashMap<>(Map.of("foo", "bar")), new HashMap<>(Map.of("baz", "qux"))))) + ); + }); } - public void testNestedGetFieldValueTypeMismatch() { + public void testNestedGetFieldValueTypeMismatch() throws Exception { try { - document.getFieldValue("foo.foo.bar", String.class); + doWithRandomAccessPattern((doc) -> doc.getFieldValue("foo.foo.bar", String.class)); } catch (IllegalArgumentException e) { assertThat( e.getMessage(), @@ -213,218 +352,408 @@ public void testNestedGetFieldValueTypeMismatch() { } } - public void testListGetFieldValue() { - assertThat(document.getFieldValue("list.0.field", String.class), equalTo("value")); + public void testListGetFieldValue() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> assertThat(doc.getFieldValue("list.0.field", String.class), equalTo("value"))); + doWithAccessPattern(FLEXIBLE, (doc) -> { + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + var illegalArgument = expectThrows(IllegalArgumentException.class, () -> doc.getFieldValue("list.0.field", String.class)); + assertThat(illegalArgument.getMessage(), equalTo("path [list.0.field] is not valid")); + illegalArgument = expectThrows( + IllegalArgumentException.class, + () -> doc.getFieldValue("dots.arrays.dotted.objects.0.foo", String.class) + ); + assertThat(illegalArgument.getMessage(), equalTo("path [dots.arrays.dotted.objects.0.foo] is not valid")); + }); } - public void testListGetFieldValueNull() { - assertThat(document.getFieldValue("list.1", String.class), nullValue()); + public void testListGetFieldValueNull() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> assertThat(doc.getFieldValue("list.1", String.class), nullValue())); + doWithAccessPattern(FLEXIBLE, (doc) -> { + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + var illegalArgument = expectThrows(IllegalArgumentException.class, () -> doc.getFieldValue("list.1", String.class)); + assertThat(illegalArgument.getMessage(), equalTo("path [list.1] is not valid")); + illegalArgument = expectThrows( + IllegalArgumentException.class, + () -> doc.getFieldValue("dots.arrays.dotted.other.0", String.class) + ); + assertThat(illegalArgument.getMessage(), equalTo("path [dots.arrays.dotted.other.0] is not valid")); + }); } - public void testListGetFieldValueIndexNotNumeric() { + public void testListGetFieldValueIndexNotNumeric() throws Exception { try { - document.getFieldValue("list.test.field", String.class); + doWithAccessPattern(CLASSIC, (doc) -> doc.getFieldValue("list.test.field", String.class)); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("[test] is not an integer, cannot be used as an index as part of path [list.test.field]")); } + try { + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> doc.getFieldValue("list.test.field", String.class)); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("path [list.test.field] is not valid")); + } + try { + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> doc.getFieldValue("dots.arrays.dotted.strings.test.field", String.class)); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("path [dots.arrays.dotted.strings.test.field] is not valid")); + } } - public void testListGetFieldValueIndexOutOfBounds() { + public void testListGetFieldValueIndexOutOfBounds() throws Exception { try { - document.getFieldValue("list.10.field", String.class); + doWithAccessPattern(CLASSIC, (doc) -> doc.getFieldValue("list.10.field", String.class)); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("[10] is out of bounds for array with length [2] as part of path [list.10.field]")); } + try { + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> doc.getFieldValue("list.10.field", String.class)); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("path [list.10.field] is not valid")); + } + try { + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> doc.getFieldValue("dots.arrays.dotted.strings.10", String.class)); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("path [dots.arrays.dotted.strings.10] is not valid")); + } } - public void testGetFieldValueNotFound() { + public void testGetFieldValueNotFound() throws Exception { try { - document.getFieldValue("not.here", String.class); + doWithAccessPattern(CLASSIC, (doc) -> doc.getFieldValue("not.here", String.class)); fail("get field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("field [not] not present as part of path [not.here]")); } + try { + doWithAccessPattern(FLEXIBLE, (doc) -> doc.getFieldValue("not.here", String.class)); + fail("get field value should have failed"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("field [not.here] not present as part of path [not.here]")); + } } - public void testGetFieldValueNotFoundNullParent() { + public void testGetFieldValueNotFoundNullParent() throws Exception { try { - document.getFieldValue("fizz.foo_null.not_there", String.class); + doWithRandomAccessPattern((doc) -> doc.getFieldValue("fizz.foo_null.not_there", String.class)); fail("get field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("cannot resolve [not_there] from null as part of path [fizz.foo_null.not_there]")); } } - public void testGetFieldValueNull() { + public void testGetFieldValueNull() throws Exception { try { - document.getFieldValue(null, String.class); + doWithRandomAccessPattern((doc) -> doc.getFieldValue(null, String.class)); fail("get field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path cannot be null nor empty")); } } - public void testGetFieldValueEmpty() { + public void testGetFieldValueEmpty() throws Exception { try { - document.getFieldValue("", String.class); + doWithRandomAccessPattern((doc) -> doc.getFieldValue("", String.class)); fail("get field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path cannot be null nor empty")); } } - public void testHasField() { - assertTrue(document.hasField("fizz")); - assertTrue(document.hasField("_index")); - assertTrue(document.hasField("_id")); - assertTrue(document.hasField("_source.fizz")); - assertTrue(document.hasField("_ingest.timestamp")); + public void testHasField() throws Exception { + doWithRandomAccessPattern((doc) -> { + assertTrue(doc.hasField("fizz")); + assertTrue(doc.hasField("_index")); + assertTrue(doc.hasField("_id")); + assertTrue(doc.hasField("_source.fizz")); + assertTrue(doc.hasField("_ingest.timestamp")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> assertTrue(doc.hasField("dotted.bar.baz"))); } - public void testHasFieldNested() { - assertTrue(document.hasField("fizz.buzz")); - assertTrue(document.hasField("_source._ingest.timestamp")); + public void testHasFieldNested() throws Exception { + doWithRandomAccessPattern((doc) -> { + assertTrue(doc.hasField("fizz.buzz")); + assertTrue(doc.hasField("_source._ingest.timestamp")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + assertTrue(doc.hasField("dots")); + { + assertFalse(doc.hasField("dots.foo")); + assertFalse(doc.hasField("dots.foo.bar")); + assertTrue(doc.hasField("dots.foo.bar.baz")); + } + + assertFalse(doc.hasField("dots.dotted")); + assertTrue(doc.hasField("dots.dotted.integers")); + { + assertTrue(doc.hasField("dots.dotted.integers.a")); + + assertFalse(doc.hasField("dots.dotted.integers.b")); + assertTrue(doc.hasField("dots.dotted.integers.b.c")); + + assertFalse(doc.hasField("dots.dotted.integers.d")); + assertFalse(doc.hasField("dots.dotted.integers.d.e")); + assertTrue(doc.hasField("dots.dotted.integers.d.e.f")); + + assertFalse(doc.hasField("dots.dotted.integers.g")); + assertFalse(doc.hasField("dots.dotted.integers.g.h")); + assertFalse(doc.hasField("dots.dotted.integers.g.h.i")); + assertTrue(doc.hasField("dots.dotted.integers.g.h.i.j")); + + assertFalse(doc.hasField("dots.dotted.integers.k")); + assertFalse(doc.hasField("dots.dotted.integers.k.l")); + assertFalse(doc.hasField("dots.dotted.integers.k.l.m")); + assertFalse(doc.hasField("dots.dotted.integers.k.l.m.n")); + assertTrue(doc.hasField("dots.dotted.integers.k.l.m.n.o")); + } + + assertTrue(doc.hasField("dots.inaccessible")); + { + assertTrue(doc.hasField("dots.inaccessible.a")); + assertTrue(doc.hasField("dots.inaccessible.a.b")); + assertTrue(doc.hasField("dots.inaccessible.a.b.c")); + } + + assertTrue(doc.hasField("dots.arrays")); + { + assertTrue(doc.hasField("dots.arrays.dotted.strings")); + assertTrue(doc.hasField("dots.arrays.dotted.objects")); + } + + assertTrue(doc.hasField("dots.single_fieldname")); + { + assertFalse(doc.hasField("dots.single_fieldname.multiple")); + assertTrue(doc.hasField("dots.single_fieldname.multiple.fieldnames")); + assertTrue(doc.hasField("dots.single_fieldname.multiple.fieldnames.single_fieldname_again")); + assertFalse(doc.hasField("dots.single_fieldname.multiple.fieldnames.single_fieldname_again.multiple")); + assertFalse(doc.hasField("dots.single_fieldname.multiple.fieldnames.single_fieldname_again.multiple.fieldnames")); + assertTrue(doc.hasField("dots.single_fieldname.multiple.fieldnames.single_fieldname_again.multiple.fieldnames.again")); + } + + assertFalse(doc.hasField("dotted.foo.bar.baz.qux")); + assertTrue(doc.hasField("dotted.foo.bar.baz.qux.quux")); + }); } - public void testListHasField() { + public void testListHasField() throws Exception { assertTrue(document.hasField("list.0.field")); + doWithAccessPattern(FLEXIBLE, (doc) -> { + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + // Until then, traversing arrays in the hasFields method returns false + assertFalse(doc.hasField("dots.arrays.dotted.strings.0")); + assertFalse(doc.hasField("dots.arrays.dotted.objects.0")); + assertFalse(doc.hasField("dots.arrays.dotted.objects.0.foo")); + }); } - public void testListHasFieldNull() { - assertTrue(document.hasField("list.1")); + public void testListHasFieldNull() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> assertTrue(doc.hasField("list.1"))); + // TODO: Flexible will have a new notation for list indexing - For now it does not locate indexed fields + doWithAccessPattern(FLEXIBLE, (doc) -> assertFalse(doc.hasField("list.1"))); + doWithAccessPattern(FLEXIBLE, (doc) -> assertFalse(doc.hasField("dots.arrays.dotted.other.0"))); } - public void testListHasFieldIndexOutOfBounds() { - assertFalse(document.hasField("list.10")); + public void testListHasFieldIndexOutOfBounds() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> assertFalse(doc.hasField("list.10"))); + // TODO: Flexible will have a new notation for list indexing - For now it does not locate indexed fields + doWithAccessPattern(FLEXIBLE, (doc) -> assertFalse(doc.hasField("list.10"))); + doWithAccessPattern(FLEXIBLE, (doc) -> assertFalse(doc.hasField("dots.arrays.dotted.strings.10"))); } - public void testListHasFieldIndexOutOfBounds_fail() { - assertTrue(document.hasField("list.0", true)); - assertTrue(document.hasField("list.1", true)); - Exception e = expectThrows(IllegalArgumentException.class, () -> document.hasField("list.2", true)); - assertThat(e.getMessage(), equalTo("[2] is out of bounds for array with length [2] as part of path [list.2]")); - e = expectThrows(IllegalArgumentException.class, () -> document.hasField("list.10", true)); - assertThat(e.getMessage(), equalTo("[10] is out of bounds for array with length [2] as part of path [list.10]")); + public void testListHasFieldIndexOutOfBounds_fail() throws Exception { + doWithAccessPattern(CLASSIC, doc -> { + assertTrue(doc.hasField("list.0", true)); + assertTrue(doc.hasField("list.1", true)); + Exception e = expectThrows(IllegalArgumentException.class, () -> doc.hasField("list.2", true)); + assertThat(e.getMessage(), equalTo("[2] is out of bounds for array with length [2] as part of path [list.2]")); + e = expectThrows(IllegalArgumentException.class, () -> doc.hasField("list.10", true)); + assertThat(e.getMessage(), equalTo("[10] is out of bounds for array with length [2] as part of path [list.10]")); + }); + doWithAccessPattern(FLEXIBLE, doc -> { + assertFalse(doc.hasField("list.0", true)); + assertFalse(doc.hasField("list.1", true)); + // TODO: Flexible will have a new notation for list indexing - we fail fast, and currently don't check the bounds + assertFalse(doc.hasField("list.2", true)); + assertFalse(doc.hasField("list.10", true)); + }); } - public void testListHasFieldIndexNotNumeric() { - assertFalse(document.hasField("list.test")); + public void testListHasFieldIndexNotNumeric() throws Exception { + doWithRandomAccessPattern((doc) -> assertFalse(doc.hasField("list.test"))); } - public void testNestedHasFieldTypeMismatch() { - assertFalse(document.hasField("foo.foo.bar")); + public void testNestedHasFieldTypeMismatch() throws Exception { + doWithRandomAccessPattern((doc) -> assertFalse(doc.hasField("foo.foo.bar"))); } - public void testHasFieldNotFound() { - assertFalse(document.hasField("not.here")); + public void testHasFieldNotFound() throws Exception { + doWithRandomAccessPattern((doc) -> assertFalse(doc.hasField("not.here"))); } - public void testHasFieldNotFoundNullParent() { - assertFalse(document.hasField("fizz.foo_null.not_there")); + public void testHasFieldNotFoundNullParent() throws Exception { + doWithRandomAccessPattern((doc) -> assertFalse(doc.hasField("fizz.foo_null.not_there"))); } - public void testHasFieldNestedNotFound() { - assertFalse(document.hasField("fizz.doesnotexist")); + public void testHasFieldNestedNotFound() throws Exception { + doWithRandomAccessPattern((doc) -> assertFalse(doc.hasField("fizz.doesnotexist"))); } - public void testHasFieldNull() { + public void testHasFieldNull() throws Exception { try { - document.hasField(null); + doWithRandomAccessPattern((doc) -> doc.hasField(null)); fail("has field should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path cannot be null nor empty")); } } - public void testHasFieldNullValue() { - assertTrue(document.hasField("fizz.foo_null")); + public void testHasFieldNullValue() throws Exception { + doWithRandomAccessPattern((doc) -> assertTrue(doc.hasField("fizz.foo_null"))); + doWithAccessPattern(FLEXIBLE, (doc) -> assertTrue(doc.hasField("dotted.bar.baz_null"))); } - public void testHasFieldEmpty() { + public void testHasFieldEmpty() throws Exception { try { - document.hasField(""); + doWithRandomAccessPattern((doc) -> doc.hasField("")); fail("has field should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path cannot be null nor empty")); } } - public void testHasFieldSourceObject() { - assertThat(document.hasField("_source"), equalTo(false)); + public void testHasFieldSourceObject() throws Exception { + doWithRandomAccessPattern((doc) -> assertThat(doc.hasField("_source"), equalTo(false))); } - public void testHasFieldIngestObject() { - assertThat(document.hasField("_ingest"), equalTo(true)); + public void testHasFieldIngestObject() throws Exception { + doWithRandomAccessPattern((doc) -> assertThat(doc.hasField("_ingest"), equalTo(true))); } - public void testHasFieldEmptyPathAfterStrippingOutPrefix() { + public void testHasFieldEmptyPathAfterStrippingOutPrefix() throws Exception { try { - document.hasField("_source."); + doWithRandomAccessPattern((doc) -> doc.hasField("_source.")); fail("has field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path [_source.] is not valid")); } try { - document.hasField("_ingest."); + doWithRandomAccessPattern((doc) -> doc.hasField("_ingest.")); fail("has field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path [_ingest.] is not valid")); } } - public void testSimpleSetFieldValue() { - document.setFieldValue("new_field", "foo"); - assertThat(document.getSourceAndMetadata().get("new_field"), equalTo("foo")); - document.setFieldValue("_ttl", "ttl"); - assertThat(document.getSourceAndMetadata().get("_ttl"), equalTo("ttl")); - document.setFieldValue("_source.another_field", "bar"); - assertThat(document.getSourceAndMetadata().get("another_field"), equalTo("bar")); - document.setFieldValue("_ingest.new_field", "new_value"); - assertThat(document.getIngestMetadata().size(), equalTo(2)); - assertThat(document.getIngestMetadata().get("new_field"), equalTo("new_value")); - document.setFieldValue("_ingest.timestamp", "timestamp"); - assertThat(document.getIngestMetadata().get("timestamp"), equalTo("timestamp")); + public void testSimpleSetFieldValue() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.setFieldValue("new_field", "foo"); + assertThat(doc.getSourceAndMetadata().get("new_field"), equalTo("foo")); + doc.setFieldValue("_ttl", "ttl"); + assertThat(doc.getSourceAndMetadata().get("_ttl"), equalTo("ttl")); + doc.setFieldValue("_source.another_field", "bar"); + assertThat(doc.getSourceAndMetadata().get("another_field"), equalTo("bar")); + doc.setFieldValue("_ingest.new_field", "new_value"); + // Metadata contains timestamp, the new_field added above, and the pipeline that is synthesized from doWithRandomAccessPattern + assertThat(doc.getIngestMetadata().size(), equalTo(3)); + assertThat(doc.getIngestMetadata().get("new_field"), equalTo("new_value")); + doc.setFieldValue("_ingest.timestamp", "timestamp"); + assertThat(doc.getIngestMetadata().get("timestamp"), equalTo("timestamp")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.setFieldValue("dotted.bar.buzz", "fizz"); + assertThat(doc.getSourceAndMetadata().get("dotted.bar.buzz"), equalTo("fizz")); + doc.setFieldValue("_source.dotted.another.buzz", "fizz"); + assertThat(doc.getSourceAndMetadata().get("dotted.another.buzz"), equalTo("fizz")); + doc.setFieldValue("_ingest.dotted.bar.buzz", "fizz"); + // Metadata contains timestamp, both fields added above, and the pipeline that is synthesized from doWithRandomAccessPattern + assertThat(doc.getIngestMetadata().size(), equalTo(4)); + assertThat(doc.getIngestMetadata().get("dotted.bar.buzz"), equalTo("fizz")); + + doc.setFieldValue("dotted.foo", "foo"); + assertThat(doc.getSourceAndMetadata().get("dotted.foo"), instanceOf(String.class)); + assertThat(doc.getSourceAndMetadata().get("dotted.foo"), equalTo("foo")); + }); } - public void testSetFieldValueNullValue() { - document.setFieldValue("new_field", (Object) null); - assertThat(document.getSourceAndMetadata().containsKey("new_field"), equalTo(true)); - assertThat(document.getSourceAndMetadata().get("new_field"), nullValue()); + public void testSetFieldValueNullValue() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.setFieldValue("new_field", (Object) null); + assertThat(doc.getSourceAndMetadata().containsKey("new_field"), equalTo(true)); + assertThat(doc.getSourceAndMetadata().get("new_field"), nullValue()); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.setFieldValue("dotted.new.field", (Object) null); + assertThat(doc.getSourceAndMetadata().containsKey("dotted.new.field"), equalTo(true)); + assertThat(doc.getSourceAndMetadata().get("dotted.new.field"), nullValue()); + }); } @SuppressWarnings("unchecked") - public void testNestedSetFieldValue() { - document.setFieldValue("a.b.c.d", "foo"); - assertThat(document.getSourceAndMetadata().get("a"), instanceOf(Map.class)); - Map a = (Map) document.getSourceAndMetadata().get("a"); - assertThat(a.get("b"), instanceOf(Map.class)); - Map b = (Map) a.get("b"); - assertThat(b.get("c"), instanceOf(Map.class)); - Map c = (Map) b.get("c"); - assertThat(c.get("d"), instanceOf(String.class)); - String d = (String) c.get("d"); - assertThat(d, equalTo("foo")); + public void testNestedSetFieldValue() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> { + doc.setFieldValue("a.b.c.d", "foo"); + assertThat(doc.getSourceAndMetadata().get("a"), instanceOf(Map.class)); + Map a = (Map) doc.getSourceAndMetadata().get("a"); + assertThat(a.get("b"), instanceOf(Map.class)); + Map b = (Map) a.get("b"); + assertThat(b.get("c"), instanceOf(Map.class)); + Map c = (Map) b.get("c"); + assertThat(c.get("d"), instanceOf(String.class)); + String d = (String) c.get("d"); + assertThat(d, equalTo("foo")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.setFieldValue("dotted.a.b.c.d", "foo"); + assertThat(doc.getSourceAndMetadata().get("dotted.a.b.c.d"), instanceOf(String.class)); + assertThat(doc.getSourceAndMetadata().get("dotted.a.b.c.d"), equalTo("foo")); + + doc.setFieldValue("dotted.foo.bar.baz.blank", "foo"); + assertThat(doc.getSourceAndMetadata().get("dotted.foo.bar.baz"), instanceOf(Map.class)); + Map dottedFooBarBaz = (Map) doc.getSourceAndMetadata().get("dotted.foo.bar.baz"); + assertThat(dottedFooBarBaz.get("blank"), instanceOf(String.class)); + assertThat(dottedFooBarBaz.get("blank"), equalTo("foo")); + }); } - public void testSetFieldValueOnExistingField() { - document.setFieldValue("foo", "newbar"); - assertThat(document.getSourceAndMetadata().get("foo"), equalTo("newbar")); + public void testSetFieldValueOnExistingField() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.setFieldValue("foo", "newbar"); + assertThat(doc.getSourceAndMetadata().get("foo"), equalTo("newbar")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.setFieldValue("dotted.bar.baz", "newbaz"); + assertThat(doc.getSourceAndMetadata().get("dotted.bar.baz"), equalTo("newbaz")); + }); } @SuppressWarnings("unchecked") - public void testSetFieldValueOnExistingParent() { - document.setFieldValue("fizz.new", "bar"); - assertThat(document.getSourceAndMetadata().get("fizz"), instanceOf(Map.class)); - Map innerMap = (Map) document.getSourceAndMetadata().get("fizz"); - assertThat(innerMap.get("new"), instanceOf(String.class)); - String value = (String) innerMap.get("new"); - assertThat(value, equalTo("bar")); + public void testSetFieldValueOnExistingParent() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.setFieldValue("fizz.new", "bar"); + assertThat(doc.getSourceAndMetadata().get("fizz"), instanceOf(Map.class)); + Map innerMap = (Map) doc.getSourceAndMetadata().get("fizz"); + assertThat(innerMap.get("new"), instanceOf(String.class)); + String value = (String) innerMap.get("new"); + assertThat(value, equalTo("bar")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.setFieldValue("dots.dotted.integers.new", "qux"); + assertThat(doc.getSourceAndMetadata().get("dots"), instanceOf(Map.class)); + Map innerMap = (Map) doc.getSourceAndMetadata().get("dots"); + assertThat(innerMap.get("dotted.integers"), instanceOf(Map.class)); + Map innermost = (Map) innerMap.get("dotted.integers"); + assertThat(innermost.get("new"), instanceOf(String.class)); + assertThat(innermost.get("new"), equalTo("qux")); + }); } - public void testSetFieldValueOnExistingParentTypeMismatch() { + public void testSetFieldValueOnExistingParentTypeMismatch() throws Exception { try { - document.setFieldValue("fizz.buzz.new", "bar"); + doWithRandomAccessPattern((doc) -> doc.setFieldValue("fizz.buzz.new", "bar")); fail("add field should have failed"); } catch (IllegalArgumentException e) { assertThat( @@ -434,503 +763,891 @@ public void testSetFieldValueOnExistingParentTypeMismatch() { } } - public void testSetFieldValueOnExistingNullParent() { + public void testSetFieldValueOnExistingNullParent() throws Exception { try { - document.setFieldValue("fizz.foo_null.test", "bar"); + doWithRandomAccessPattern((doc) -> doc.setFieldValue("fizz.foo_null.test", "bar")); fail("add field should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("cannot set [test] with null parent as part of path [fizz.foo_null.test]")); } } - public void testSetFieldValueNullName() { + public void testSetFieldValueNullName() throws Exception { try { - document.setFieldValue(null, "bar"); + doWithRandomAccessPattern((doc) -> doc.setFieldValue(null, "bar")); fail("add field should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path cannot be null nor empty")); } } - public void testSetSourceObject() { - document.setFieldValue("_source", "value"); - assertThat(document.getSourceAndMetadata().get("_source"), equalTo("value")); + public void testSetSourceObject() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.setFieldValue("_source", "value"); + assertThat(doc.getSourceAndMetadata().get("_source"), equalTo("value")); + }); } - public void testSetIngestObject() { - document.setFieldValue("_ingest", "value"); - assertThat(document.getSourceAndMetadata().get("_ingest"), equalTo("value")); + public void testSetIngestObject() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.setFieldValue("_ingest", "value"); + assertThat(doc.getSourceAndMetadata().get("_ingest"), equalTo("value")); + }); } - public void testSetIngestSourceObject() { - // test that we don't strip out the _source prefix when _ingest is used - document.setFieldValue("_ingest._source", "value"); - assertThat(document.getIngestMetadata().get("_source"), equalTo("value")); + public void testSetIngestSourceObject() throws Exception { + doWithRandomAccessPattern((doc) -> { + // test that we don't strip out the _source prefix when _ingest is used + doc.setFieldValue("_ingest._source", "value"); + assertThat(doc.getIngestMetadata().get("_source"), equalTo("value")); + }); } - public void testSetEmptyPathAfterStrippingOutPrefix() { + public void testSetEmptyPathAfterStrippingOutPrefix() throws Exception { try { - document.setFieldValue("_source.", "value"); + doWithRandomAccessPattern((doc) -> doc.setFieldValue("_source.", "value")); fail("set field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path [_source.] is not valid")); } try { - document.setFieldValue("_ingest.", "_value"); + doWithRandomAccessPattern((doc) -> doc.setFieldValue("_ingest.", "_value")); fail("set field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path [_ingest.] is not valid")); } } - public void testListSetFieldValueNoIndexProvided() { - document.setFieldValue("list", "value"); - Object object = document.getSourceAndMetadata().get("list"); - assertThat(object, instanceOf(String.class)); - assertThat(object, equalTo("value")); - } - - public void testListAppendFieldValue() { - document.appendFieldValue("list", "new_value"); - Object object = document.getSourceAndMetadata().get("list"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(3)); - assertThat(list.get(0), equalTo(Map.of("field", "value"))); - assertThat(list.get(1), nullValue()); - assertThat(list.get(2), equalTo("new_value")); - } - - public void testListAppendFieldValueWithDuplicate() { - document.appendFieldValue("list2", "foo", false); - Object object = document.getSourceAndMetadata().get("list2"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(3)); - assertThat(list, equalTo(List.of("foo", "bar", "baz"))); - } - - public void testListAppendFieldValueWithoutDuplicate() { - document.appendFieldValue("list2", "foo2", false); - Object object = document.getSourceAndMetadata().get("list2"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(4)); - assertThat(list, equalTo(List.of("foo", "bar", "baz", "foo2"))); - } - - public void testListAppendFieldValues() { - document.appendFieldValue("list", List.of("item1", "item2", "item3")); - Object object = document.getSourceAndMetadata().get("list"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(5)); - assertThat(list.get(0), equalTo(Map.of("field", "value"))); - assertThat(list.get(1), nullValue()); - assertThat(list.get(2), equalTo("item1")); - assertThat(list.get(3), equalTo("item2")); - assertThat(list.get(4), equalTo("item3")); - } - - public void testListAppendFieldValuesWithoutDuplicates() { - document.appendFieldValue("list2", List.of("foo", "bar", "baz", "foo2"), false); - Object object = document.getSourceAndMetadata().get("list2"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(4)); - assertThat(list.get(0), equalTo("foo")); - assertThat(list.get(1), equalTo("bar")); - assertThat(list.get(2), equalTo("baz")); - assertThat(list.get(3), equalTo("foo2")); - } - - public void testAppendFieldValueToNonExistingList() { - document.appendFieldValue("non_existing_list", "new_value"); - Object object = document.getSourceAndMetadata().get("non_existing_list"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(1)); - assertThat(list.get(0), equalTo("new_value")); - } - - public void testAppendFieldValuesToNonExistingList() { - document.appendFieldValue("non_existing_list", List.of("item1", "item2", "item3")); - Object object = document.getSourceAndMetadata().get("non_existing_list"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(3)); - assertThat(list.get(0), equalTo("item1")); - assertThat(list.get(1), equalTo("item2")); - assertThat(list.get(2), equalTo("item3")); - } - - public void testAppendFieldValueConvertStringToList() { - document.appendFieldValue("fizz.buzz", "new_value"); - Object object = document.getSourceAndMetadata().get("fizz"); - assertThat(object, instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map map = (Map) object; - object = map.get("buzz"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(2)); - assertThat(list.get(0), equalTo("hello world")); - assertThat(list.get(1), equalTo("new_value")); - } - - public void testAppendFieldValuesConvertStringToList() { - document.appendFieldValue("fizz.buzz", List.of("item1", "item2", "item3")); - Object object = document.getSourceAndMetadata().get("fizz"); - assertThat(object, instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map map = (Map) object; - object = map.get("buzz"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(4)); - assertThat(list.get(0), equalTo("hello world")); - assertThat(list.get(1), equalTo("item1")); - assertThat(list.get(2), equalTo("item2")); - assertThat(list.get(3), equalTo("item3")); - } - - public void testAppendFieldValueConvertIntegerToList() { - document.appendFieldValue("int", 456); - Object object = document.getSourceAndMetadata().get("int"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(2)); - assertThat(list.get(0), equalTo(123)); - assertThat(list.get(1), equalTo(456)); - } - - public void testAppendFieldValuesConvertIntegerToList() { - document.appendFieldValue("int", List.of(456, 789)); - Object object = document.getSourceAndMetadata().get("int"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(3)); - assertThat(list.get(0), equalTo(123)); - assertThat(list.get(1), equalTo(456)); - assertThat(list.get(2), equalTo(789)); - } - - public void testAppendFieldValueConvertMapToList() { - document.appendFieldValue("fizz", Map.of("field", "value")); - Object object = document.getSourceAndMetadata().get("fizz"); - assertThat(object, instanceOf(List.class)); - List list = (List) object; - assertThat(list.size(), equalTo(2)); - assertThat(list.get(0), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map map = (Map) list.get(0); - assertThat(map.size(), equalTo(4)); - assertThat(list.get(1), equalTo(Map.of("field", "value"))); - } - - public void testAppendFieldValueToNull() { - document.appendFieldValue("fizz.foo_null", "new_value"); - Object object = document.getSourceAndMetadata().get("fizz"); - assertThat(object, instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map map = (Map) object; - object = map.get("foo_null"); - assertThat(object, instanceOf(List.class)); - List list = (List) object; - assertThat(list.size(), equalTo(2)); - assertThat(list.get(0), nullValue()); - assertThat(list.get(1), equalTo("new_value")); - } - - public void testAppendFieldValueToListElement() { - document.appendFieldValue("fizz.list.0", "item2"); - Object object = document.getSourceAndMetadata().get("fizz"); - assertThat(object, instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map map = (Map) object; - object = map.get("list"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(1)); - object = list.get(0); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List innerList = (List) object; - assertThat(innerList.size(), equalTo(2)); - assertThat(innerList.get(0), equalTo("item1")); - assertThat(innerList.get(1), equalTo("item2")); - } - - public void testAppendFieldValuesToListElement() { - document.appendFieldValue("fizz.list.0", List.of("item2", "item3", "item4")); - Object object = document.getSourceAndMetadata().get("fizz"); - assertThat(object, instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map map = (Map) object; - object = map.get("list"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(1)); - object = list.get(0); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List innerList = (List) object; - assertThat(innerList.size(), equalTo(4)); - assertThat(innerList.get(0), equalTo("item1")); - assertThat(innerList.get(1), equalTo("item2")); - assertThat(innerList.get(2), equalTo("item3")); - assertThat(innerList.get(3), equalTo("item4")); - } - - public void testAppendFieldValueConvertStringListElementToList() { - document.appendFieldValue("fizz.list.0.0", "new_value"); - Object object = document.getSourceAndMetadata().get("fizz"); - assertThat(object, instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map map = (Map) object; - object = map.get("list"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(1)); - object = list.get(0); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List innerList = (List) object; - object = innerList.get(0); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List innerInnerList = (List) object; - assertThat(innerInnerList.size(), equalTo(2)); - assertThat(innerInnerList.get(0), equalTo("item1")); - assertThat(innerInnerList.get(1), equalTo("new_value")); - } - - public void testAppendFieldValuesConvertStringListElementToList() { - document.appendFieldValue("fizz.list.0.0", List.of("item2", "item3", "item4")); - Object object = document.getSourceAndMetadata().get("fizz"); - assertThat(object, instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map map = (Map) object; - object = map.get("list"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(1)); - object = list.get(0); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List innerList = (List) object; - object = innerList.get(0); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List innerInnerList = (List) object; - assertThat(innerInnerList.size(), equalTo(4)); - assertThat(innerInnerList.get(0), equalTo("item1")); - assertThat(innerInnerList.get(1), equalTo("item2")); - assertThat(innerInnerList.get(2), equalTo("item3")); - assertThat(innerInnerList.get(3), equalTo("item4")); - } - - public void testAppendFieldValueListElementConvertMapToList() { - document.appendFieldValue("list.0", Map.of("item2", "value2")); - Object object = document.getSourceAndMetadata().get("list"); - assertThat(object, instanceOf(List.class)); - List list = (List) object; - assertThat(list.size(), equalTo(2)); - assertThat(list.get(0), instanceOf(List.class)); - assertThat(list.get(1), nullValue()); - list = (List) list.get(0); - assertThat(list.size(), equalTo(2)); - assertThat(list.get(0), equalTo(Map.of("field", "value"))); - assertThat(list.get(1), equalTo(Map.of("item2", "value2"))); - } - - public void testAppendFieldValueToNullListElement() { - document.appendFieldValue("list.1", "new_value"); - Object object = document.getSourceAndMetadata().get("list"); - assertThat(object, instanceOf(List.class)); - List list = (List) object; - assertThat(list.get(1), instanceOf(List.class)); - list = (List) list.get(1); - assertThat(list.size(), equalTo(2)); - assertThat(list.get(0), nullValue()); - assertThat(list.get(1), equalTo("new_value")); - } - - public void testAppendFieldValueToListOfMaps() { - document.appendFieldValue("list", Map.of("item2", "value2")); - Object object = document.getSourceAndMetadata().get("list"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(3)); - assertThat(list.get(0), equalTo(Map.of("field", "value"))); - assertThat(list.get(1), nullValue()); - assertThat(list.get(2), equalTo(Map.of("item2", "value2"))); - } - - public void testListSetFieldValueIndexProvided() { - document.setFieldValue("list.1", "value"); - Object object = document.getSourceAndMetadata().get("list"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(2)); - assertThat(list.get(0), equalTo(Map.of("field", "value"))); - assertThat(list.get(1), equalTo("value")); - } - - public void testSetFieldValueListAsPartOfPath() { - document.setFieldValue("list.0.field", "new_value"); - Object object = document.getSourceAndMetadata().get("list"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(2)); - assertThat(list.get(0), equalTo(Map.of("field", "new_value"))); - assertThat(list.get(1), nullValue()); - } - - public void testListSetFieldValueIndexNotNumeric() { + public void testListSetFieldValueNoIndexProvided() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.setFieldValue("list", "value"); + Object object = doc.getSourceAndMetadata().get("list"); + assertThat(object, instanceOf(String.class)); + assertThat(object, equalTo("value")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.setFieldValue("dots.arrays.dotted.strings", "value"); + Object object = doc.getSourceAndMetadata().get("dots"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object arraysField = ((Map) object).get("arrays"); + assertThat(arraysField, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object dottedStringsField = ((Map) arraysField).get("dotted.strings"); + assertThat(dottedStringsField, instanceOf(String.class)); + assertThat(dottedStringsField, equalTo("value")); + }); + } + + public void testListAppendFieldValue() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("list", "new_value"); + Object object = doc.getSourceAndMetadata().get("list"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(3)); + assertThat(list.get(0), equalTo(Map.of("field", "value"))); + assertThat(list.get(1), nullValue()); + assertThat(list.get(2), equalTo("new_value")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.arrays.dotted.strings", "value"); + Object object = doc.getSourceAndMetadata().get("dots"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object arraysField = ((Map) object).get("arrays"); + assertThat(arraysField, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object dottedStringsField = ((Map) arraysField).get("dotted.strings"); + assertThat(dottedStringsField, instanceOf(List.class)); + assertThat(dottedStringsField, equalTo(List.of("a", "b", "c", "d", "value"))); + }); + } + + public void testListAppendFieldValueWithDuplicate() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("list2", "foo", false); + Object object = doc.getSourceAndMetadata().get("list2"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(3)); + assertThat(list, equalTo(List.of("foo", "bar", "baz"))); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.arrays.dotted.strings", "a", false); + Object object = doc.getSourceAndMetadata().get("dots"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object arraysField = ((Map) object).get("arrays"); + assertThat(arraysField, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object dottedStringsField = ((Map) arraysField).get("dotted.strings"); + assertThat(dottedStringsField, instanceOf(List.class)); + assertThat(dottedStringsField, equalTo(List.of("a", "b", "c", "d"))); + }); + } + + public void testListAppendFieldValueWithoutDuplicate() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("list2", "foo2", false); + Object object = doc.getSourceAndMetadata().get("list2"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(4)); + assertThat(list, equalTo(List.of("foo", "bar", "baz", "foo2"))); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.arrays.dotted.strings", "e", false); + Object object = doc.getSourceAndMetadata().get("dots"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object arraysField = ((Map) object).get("arrays"); + assertThat(arraysField, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object dottedStringsField = ((Map) arraysField).get("dotted.strings"); + assertThat(dottedStringsField, instanceOf(List.class)); + assertThat(dottedStringsField, equalTo(List.of("a", "b", "c", "d", "e"))); + }); + } + + public void testListAppendFieldValues() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("list", List.of("item1", "item2", "item3")); + Object object = doc.getSourceAndMetadata().get("list"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(5)); + assertThat(list.get(0), equalTo(Map.of("field", "value"))); + assertThat(list.get(1), nullValue()); + assertThat(list.get(2), equalTo("item1")); + assertThat(list.get(3), equalTo("item2")); + assertThat(list.get(4), equalTo("item3")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.arrays.dotted.strings", List.of("e", "f", "g")); + Object object = doc.getSourceAndMetadata().get("dots"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object arraysField = ((Map) object).get("arrays"); + assertThat(arraysField, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object dottedStringsField = ((Map) arraysField).get("dotted.strings"); + assertThat(dottedStringsField, instanceOf(List.class)); + assertThat(dottedStringsField, equalTo(List.of("a", "b", "c", "d", "e", "f", "g"))); + }); + } + + public void testListAppendFieldValuesWithoutDuplicates() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("list2", List.of("foo", "bar", "baz", "foo2"), false); + Object object = doc.getSourceAndMetadata().get("list2"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(4)); + assertThat(list.get(0), equalTo("foo")); + assertThat(list.get(1), equalTo("bar")); + assertThat(list.get(2), equalTo("baz")); + assertThat(list.get(3), equalTo("foo2")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.arrays.dotted.strings", List.of("a", "b", "c", "d", "e"), false); + Object object = doc.getSourceAndMetadata().get("dots"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object arraysField = ((Map) object).get("arrays"); + assertThat(arraysField, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object dottedStringsField = ((Map) arraysField).get("dotted.strings"); + assertThat(dottedStringsField, instanceOf(List.class)); + assertThat(dottedStringsField, equalTo(List.of("a", "b", "c", "d", "e"))); + }); + } + + public void testAppendFieldValueToNonExistingList() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("non_existing_list", "new_value"); + Object object = doc.getSourceAndMetadata().get("non_existing_list"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(1)); + assertThat(list.get(0), equalTo("new_value")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.arrays.dotted.missing", "a"); + Object object = doc.getSourceAndMetadata().get("dots"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object arraysField = ((Map) object).get("arrays"); + assertThat(arraysField, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object dottedStringsField = ((Map) arraysField).get("dotted.missing"); + assertThat(dottedStringsField, instanceOf(List.class)); + assertThat(dottedStringsField, equalTo(List.of("a"))); + }); + } + + public void testAppendFieldValuesToNonExistingList() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("non_existing_list", List.of("item1", "item2", "item3")); + Object object = doc.getSourceAndMetadata().get("non_existing_list"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(3)); + assertThat(list.get(0), equalTo("item1")); + assertThat(list.get(1), equalTo("item2")); + assertThat(list.get(2), equalTo("item3")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.arrays.dotted.missing", List.of("a", "b", "c")); + Object object = doc.getSourceAndMetadata().get("dots"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object arraysField = ((Map) object).get("arrays"); + assertThat(arraysField, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object dottedStringsField = ((Map) arraysField).get("dotted.missing"); + assertThat(dottedStringsField, instanceOf(List.class)); + assertThat(dottedStringsField, equalTo(List.of("a", "b", "c"))); + }); + } + + public void testAppendFieldValueConvertStringToList() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("fizz.buzz", "new_value"); + Object object = doc.getSourceAndMetadata().get("fizz"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map map = (Map) object; + object = map.get("buzz"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(2)); + assertThat(list.get(0), equalTo("hello world")); + assertThat(list.get(1), equalTo("new_value")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.foo.bar.baz", "new_value"); + Object object = doc.getSourceAndMetadata().get("dots"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object foobarbaz = ((Map) object).get("foo.bar.baz"); + assertThat(foobarbaz, instanceOf(List.class)); + assertThat(foobarbaz, equalTo(List.of("fizzbuzz", "new_value"))); + }); + } + + public void testAppendFieldValuesConvertStringToList() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("fizz.buzz", List.of("item1", "item2", "item3")); + Object object = doc.getSourceAndMetadata().get("fizz"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map map = (Map) object; + object = map.get("buzz"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(4)); + assertThat(list.get(0), equalTo("hello world")); + assertThat(list.get(1), equalTo("item1")); + assertThat(list.get(2), equalTo("item2")); + assertThat(list.get(3), equalTo("item3")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.foo.bar.baz", List.of("fizz", "buzz", "quack")); + Object object = doc.getSourceAndMetadata().get("dots"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object foobarbaz = ((Map) object).get("foo.bar.baz"); + assertThat(foobarbaz, instanceOf(List.class)); + assertThat(foobarbaz, equalTo(List.of("fizzbuzz", "fizz", "buzz", "quack"))); + }); + } + + public void testAppendFieldValueConvertIntegerToList() throws Exception { + doWithRandomAccessPattern((doc) -> { + document.appendFieldValue("int", 456); + Object object = document.getSourceAndMetadata().get("int"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(2)); + assertThat(list.get(0), equalTo(123)); + assertThat(list.get(1), equalTo(456)); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.dotted.integers.a", 2); + Object dots = doc.getSourceAndMetadata().get("dots"); + assertThat(dots, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object dottedIntegers = ((Map) dots).get("dotted.integers"); + assertThat(dottedIntegers, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object a = ((Map) dottedIntegers).get("a"); + assertThat(a, instanceOf(List.class)); + assertThat(a, equalTo(List.of(1, 2))); + }); + } + + public void testAppendFieldValuesConvertIntegerToList() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("int", List.of(456, 789)); + Object object = doc.getSourceAndMetadata().get("int"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(3)); + assertThat(list.get(0), equalTo(123)); + assertThat(list.get(1), equalTo(456)); + assertThat(list.get(2), equalTo(789)); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.dotted.integers.a", List.of(2, 3)); + Object dots = doc.getSourceAndMetadata().get("dots"); + assertThat(dots, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object dottedIntegers = ((Map) dots).get("dotted.integers"); + assertThat(dottedIntegers, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object a = ((Map) dottedIntegers).get("a"); + assertThat(a, instanceOf(List.class)); + assertThat(a, equalTo(List.of(1, 2, 3))); + }); + } + + public void testAppendFieldValueConvertMapToList() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("fizz", Map.of("field", "value")); + Object object = doc.getSourceAndMetadata().get("fizz"); + assertThat(object, instanceOf(List.class)); + List list = (List) object; + assertThat(list.size(), equalTo(2)); + assertThat(list.get(0), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map map = (Map) list.get(0); + assertThat(map.size(), equalTo(4)); + assertThat(list.get(1), equalTo(Map.of("field", "value"))); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.dotted.integers", Map.of("x", "y")); + Object dots = doc.getSourceAndMetadata().get("dots"); + assertThat(dots, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object dottedIntegers = ((Map) dots).get("dotted.integers"); + assertThat(dottedIntegers, instanceOf(List.class)); + List dottedIntegersList = (List) dottedIntegers; + assertThat(dottedIntegersList.size(), equalTo(2)); + assertThat(dottedIntegersList.get(0), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map originalMap = (Map) dottedIntegersList.get(0); + assertThat(originalMap.size(), equalTo(5)); // 5 entries in the original map + assertThat(dottedIntegersList.get(1), equalTo(Map.of("x", "y"))); + }); + } + + public void testAppendFieldValueToNull() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("fizz.foo_null", "new_value"); + Object object = doc.getSourceAndMetadata().get("fizz"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map map = (Map) object; + object = map.get("foo_null"); + assertThat(object, instanceOf(List.class)); + List list = (List) object; + assertThat(list.size(), equalTo(2)); + assertThat(list.get(0), nullValue()); + assertThat(list.get(1), equalTo("new_value")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dotted.bar.baz_null", "new_value"); + Object object = doc.getSourceAndMetadata().get("dotted.bar.baz_null"); + assertThat(object, instanceOf(List.class)); + List list = (List) object; + assertThat(list.size(), equalTo(2)); + assertThat(list.get(0), nullValue()); + assertThat(list.get(1), equalTo("new_value")); + }); + } + + public void testAppendFieldValueToListElement() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> { + doc.appendFieldValue("fizz.list.0", "item2"); + Object object = doc.getSourceAndMetadata().get("fizz"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map map = (Map) object; + object = map.get("list"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(1)); + object = list.get(0); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List innerList = (List) object; + assertThat(innerList.size(), equalTo(2)); + assertThat(innerList.get(0), equalTo("item1")); + assertThat(innerList.get(1), equalTo("item2")); + }); + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> { + var illegalArgument = expectThrows( + IllegalArgumentException.class, + () -> doc.appendFieldValue("dots.arrays.dotted.strings.0", "a1") + ); + assertThat(illegalArgument.getMessage(), equalTo("path [dots.arrays.dotted.strings.0] is not valid")); + }); + } + + public void testAppendFieldValuesToListElement() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> { + doc.appendFieldValue("fizz.list.0", List.of("item2", "item3", "item4")); + Object object = doc.getSourceAndMetadata().get("fizz"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map map = (Map) object; + object = map.get("list"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(1)); + object = list.get(0); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List innerList = (List) object; + assertThat(innerList.size(), equalTo(4)); + assertThat(innerList.get(0), equalTo("item1")); + assertThat(innerList.get(1), equalTo("item2")); + assertThat(innerList.get(2), equalTo("item3")); + assertThat(innerList.get(3), equalTo("item4")); + }); + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> { + var illegalArgument = expectThrows( + IllegalArgumentException.class, + () -> doc.appendFieldValue("dots.arrays.dotted.strings.0", List.of("a1", "a2", "a3")) + ); + assertThat(illegalArgument.getMessage(), equalTo("path [dots.arrays.dotted.strings.0] is not valid")); + }); + } + + public void testAppendFieldValueConvertStringListElementToList() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> { + doc.appendFieldValue("fizz.list.0.0", "new_value"); + Object object = doc.getSourceAndMetadata().get("fizz"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map map = (Map) object; + object = map.get("list"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(1)); + object = list.get(0); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List innerList = (List) object; + object = innerList.get(0); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List innerInnerList = (List) object; + assertThat(innerInnerList.size(), equalTo(2)); + assertThat(innerInnerList.get(0), equalTo("item1")); + assertThat(innerInnerList.get(1), equalTo("new_value")); + }); + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> { + var illegalArgument = expectThrows(IllegalArgumentException.class, () -> doc.appendFieldValue("fizz.list.0.0", "new_value")); + assertThat(illegalArgument.getMessage(), equalTo("path [fizz.list.0.0] is not valid")); + }); + } + + public void testAppendFieldValuesConvertStringListElementToList() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> { + doc.appendFieldValue("fizz.list.0.0", List.of("item2", "item3", "item4")); + Object object = doc.getSourceAndMetadata().get("fizz"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map map = (Map) object; + object = map.get("list"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(1)); + object = list.get(0); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List innerList = (List) object; + object = innerList.get(0); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List innerInnerList = (List) object; + assertThat(innerInnerList.size(), equalTo(4)); + assertThat(innerInnerList.get(0), equalTo("item1")); + assertThat(innerInnerList.get(1), equalTo("item2")); + assertThat(innerInnerList.get(2), equalTo("item3")); + assertThat(innerInnerList.get(3), equalTo("item4")); + }); + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> { + var illegalArgument = expectThrows( + IllegalArgumentException.class, + () -> doc.appendFieldValue("fizz.list.0.0", List.of("item2", "item3", "item4")) + ); + assertThat(illegalArgument.getMessage(), equalTo("path [fizz.list.0.0] is not valid")); + }); + } + + public void testAppendFieldValueListElementConvertMapToList() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> { + doc.appendFieldValue("list.0", Map.of("item2", "value2")); + Object object = doc.getSourceAndMetadata().get("list"); + assertThat(object, instanceOf(List.class)); + List list = (List) object; + assertThat(list.size(), equalTo(2)); + assertThat(list.get(0), instanceOf(List.class)); + assertThat(list.get(1), nullValue()); + list = (List) list.get(0); + assertThat(list.size(), equalTo(2)); + assertThat(list.get(0), equalTo(Map.of("field", "value"))); + assertThat(list.get(1), equalTo(Map.of("item2", "value2"))); + }); + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> { + var illegalArgument = expectThrows( + IllegalArgumentException.class, + () -> doc.appendFieldValue("list.0", Map.of("item2", "value2")) + ); + assertThat(illegalArgument.getMessage(), equalTo("path [list.0] is not valid")); + }); + } + + public void testAppendFieldValueToNullListElement() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> { + doc.appendFieldValue("list.1", "new_value"); + Object object = doc.getSourceAndMetadata().get("list"); + assertThat(object, instanceOf(List.class)); + List list = (List) object; + assertThat(list.get(1), instanceOf(List.class)); + list = (List) list.get(1); + assertThat(list.size(), equalTo(2)); + assertThat(list.get(0), nullValue()); + assertThat(list.get(1), equalTo("new_value")); + }); + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> { + var illegalArgument = expectThrows(IllegalArgumentException.class, () -> doc.appendFieldValue("list.1", "new_value")); + assertThat(illegalArgument.getMessage(), equalTo("path [list.1] is not valid")); + }); + } + + public void testAppendFieldValueToListOfMaps() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.appendFieldValue("list", Map.of("item2", "value2")); + Object object = doc.getSourceAndMetadata().get("list"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(3)); + assertThat(list.get(0), equalTo(Map.of("field", "value"))); + assertThat(list.get(1), nullValue()); + assertThat(list.get(2), equalTo(Map.of("item2", "value2"))); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.appendFieldValue("dots.arrays.dotted.objects", Map.of("item2", "value2")); + Object object = doc.getSourceAndMetadata().get("dots"); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object arrays = ((Map) object).get("arrays"); + assertThat(arrays, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Object dottedObjects = ((Map) arrays).get("dotted.objects"); + assertThat(dottedObjects, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) dottedObjects; + assertThat(list.size(), equalTo(3)); + assertThat(list.get(0), equalTo(Map.of("foo", "bar"))); + assertThat(list.get(1), equalTo(Map.of("baz", "qux"))); + assertThat(list.get(2), equalTo(Map.of("item2", "value2"))); + }); + } + + public void testListSetFieldValueIndexProvided() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> { + doc.setFieldValue("list.1", "value"); + Object object = doc.getSourceAndMetadata().get("list"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(2)); + assertThat(list.get(0), equalTo(Map.of("field", "value"))); + assertThat(list.get(1), equalTo("value")); + }); + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> { + var illegalArgument = expectThrows(IllegalArgumentException.class, () -> doc.setFieldValue("list.1", "value")); + assertThat(illegalArgument.getMessage(), equalTo("path [list.1] is not valid")); + }); + } + + public void testSetFieldValueListAsPartOfPath() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> { + doc.setFieldValue("list.0.field", "new_value"); + Object object = doc.getSourceAndMetadata().get("list"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(2)); + assertThat(list.get(0), equalTo(Map.of("field", "new_value"))); + assertThat(list.get(1), nullValue()); + }); + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> { + var illegalArgument = expectThrows(IllegalArgumentException.class, () -> doc.setFieldValue("list.0.field", "new_value")); + assertThat(illegalArgument.getMessage(), equalTo("path [list.0.field] is not valid")); + }); + } + + public void testListSetFieldValueIndexNotNumeric() throws Exception { try { - document.setFieldValue("list.test", "value"); + doWithAccessPattern(CLASSIC, (doc) -> doc.setFieldValue("list.test", "value")); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("[test] is not an integer, cannot be used as an index as part of path [list.test]")); } try { - document.setFieldValue("list.test.field", "new_value"); + doWithAccessPattern(CLASSIC, (doc) -> doc.setFieldValue("list.test.field", "new_value")); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("[test] is not an integer, cannot be used as an index as part of path [list.test.field]")); } + + try { + doWithAccessPattern(FLEXIBLE, (doc) -> doc.setFieldValue("list.test", "value")); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("path [list.test] is not valid")); + } + + try { + doWithAccessPattern(FLEXIBLE, (doc) -> doc.setFieldValue("list.test.field", "new_value")); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("path [list.test.field] is not valid")); + } } - public void testListSetFieldValueIndexOutOfBounds() { + public void testListSetFieldValueIndexOutOfBounds() throws Exception { try { - document.setFieldValue("list.10", "value"); + doWithAccessPattern(CLASSIC, (doc) -> doc.setFieldValue("list.10", "value")); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("[10] is out of bounds for array with length [2] as part of path [list.10]")); } try { - document.setFieldValue("list.10.field", "value"); + doWithAccessPattern(CLASSIC, (doc) -> doc.setFieldValue("list.10.field", "value")); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("[10] is out of bounds for array with length [2] as part of path [list.10.field]")); } + + try { + doWithAccessPattern(FLEXIBLE, (doc) -> doc.setFieldValue("list.10", "value")); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("path [list.10] is not valid")); + } + + try { + doWithAccessPattern(FLEXIBLE, (doc) -> doc.setFieldValue("list.10.field", "value")); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("path [list.10.field] is not valid")); + } } - public void testSetFieldValueEmptyName() { + public void testSetFieldValueEmptyName() throws Exception { try { - document.setFieldValue("", "bar"); + doWithRandomAccessPattern((doc) -> doc.setFieldValue("", "bar")); fail("add field should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path cannot be null nor empty")); } } - public void testRemoveField() { - document.removeField("foo"); - assertThat(document.getSourceAndMetadata().size(), equalTo(10)); - assertThat(document.getSourceAndMetadata().containsKey("foo"), equalTo(false)); - document.removeField("_index"); - assertThat(document.getSourceAndMetadata().size(), equalTo(9)); - assertThat(document.getSourceAndMetadata().containsKey("_index"), equalTo(false)); - document.removeField("_source.fizz"); - assertThat(document.getSourceAndMetadata().size(), equalTo(8)); - assertThat(document.getSourceAndMetadata().containsKey("fizz"), equalTo(false)); - assertThat(document.getIngestMetadata().size(), equalTo(1)); - document.removeField("_ingest.timestamp"); - assertThat(document.getSourceAndMetadata().size(), equalTo(8)); - assertThat(document.getIngestMetadata().size(), equalTo(0)); + public void testRemoveField() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.removeField("foo"); + assertThat(doc.getSourceAndMetadata().size(), equalTo(14)); + assertThat(doc.getSourceAndMetadata().containsKey("foo"), equalTo(false)); + doc.removeField("_index"); + assertThat(doc.getSourceAndMetadata().size(), equalTo(13)); + assertThat(doc.getSourceAndMetadata().containsKey("_index"), equalTo(false)); + doc.removeField("_source.fizz"); + assertThat(doc.getSourceAndMetadata().size(), equalTo(12)); + assertThat(doc.getSourceAndMetadata().containsKey("fizz"), equalTo(false)); + assertThat(doc.getIngestMetadata().size(), equalTo(2)); + doc.removeField("_ingest.timestamp"); + assertThat(doc.getSourceAndMetadata().size(), equalTo(12)); + assertThat(doc.getIngestMetadata().size(), equalTo(1)); + doc.removeField("_ingest.pipeline"); + assertThat(doc.getSourceAndMetadata().size(), equalTo(12)); + assertThat(doc.getIngestMetadata().size(), equalTo(0)); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.removeField("dotted.bar.baz"); + assertThat(doc.getSourceAndMetadata().size(), equalTo(11)); + assertThat(doc.getSourceAndMetadata().containsKey("dotted.bar.baz"), equalTo(false)); + }); } - public void testRemoveFieldIgnoreMissing() { - document.removeField("foo", randomBoolean()); - assertThat(document.getSourceAndMetadata().size(), equalTo(10)); - assertThat(document.getSourceAndMetadata().containsKey("foo"), equalTo(false)); - document.removeField("_index", randomBoolean()); - assertThat(document.getSourceAndMetadata().size(), equalTo(9)); - assertThat(document.getSourceAndMetadata().containsKey("_index"), equalTo(false)); + public void testRemoveFieldIgnoreMissing() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.removeField("foo", randomBoolean()); + assertThat(doc.getSourceAndMetadata().size(), equalTo(14)); + assertThat(doc.getSourceAndMetadata().containsKey("foo"), equalTo(false)); + doc.removeField("_index", randomBoolean()); + assertThat(doc.getSourceAndMetadata().size(), equalTo(13)); + assertThat(doc.getSourceAndMetadata().containsKey("_index"), equalTo(false)); + }); // if ignoreMissing is false, we throw an exception for values that aren't found - IllegalArgumentException e; switch (randomIntBetween(0, 2)) { - case 0 -> { - document.setFieldValue("fizz.some", (Object) null); - e = expectThrows(IllegalArgumentException.class, () -> document.removeField("fizz.some.nonsense", false)); + case 0 -> doWithRandomAccessPattern((doc) -> { + doc.setFieldValue("fizz.some", (Object) null); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> doc.removeField("fizz.some.nonsense", false) + ); assertThat(e.getMessage(), is("cannot remove [nonsense] from null as part of path [fizz.some.nonsense]")); - } + }); case 1 -> { - document.setFieldValue("fizz.some", List.of("foo", "bar")); - e = expectThrows(IllegalArgumentException.class, () -> document.removeField("fizz.some.nonsense", false)); - assertThat( - e.getMessage(), - is("[nonsense] is not an integer, cannot be used as an index as part of path [fizz.some.nonsense]") - ); + // Different error messages for each access pattern when trying to remove an element from a list incorrectly + doWithAccessPattern(CLASSIC, (doc) -> { + doc.setFieldValue("fizz.some", List.of("foo", "bar")); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> doc.removeField("fizz.some.nonsense", false) + ); + assertThat( + e.getMessage(), + is("[nonsense] is not an integer, cannot be used as an index as part of path [fizz.some.nonsense]") + ); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.setFieldValue("fizz.other", List.of("foo", "bar")); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> doc.removeField("fizz.other.nonsense", false) + ); + assertThat(e.getMessage(), is("path [fizz.other.nonsense] is not valid")); + }); } case 2 -> { - e = expectThrows(IllegalArgumentException.class, () -> document.removeField("fizz.some.nonsense", false)); - assertThat(e.getMessage(), is("field [some] not present as part of path [fizz.some.nonsense]")); + // Different error messages when removing a nested field that does not exist + doWithAccessPattern(CLASSIC, (doc) -> { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> doc.removeField("fizz.some.nonsense", false) + ); + assertThat(e.getMessage(), is("field [some] not present as part of path [fizz.some.nonsense]")); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> doc.removeField("fizz.some.nonsense", false) + ); + assertThat(e.getMessage(), is("field [some.nonsense] not present as part of path [fizz.some.nonsense]")); + }); } default -> throw new AssertionError("failure, got illegal switch case"); } // but no exception is thrown if ignoreMissing is true - document.removeField("fizz.some.nonsense", true); + doWithRandomAccessPattern((doc) -> doc.removeField("fizz.some.nonsense", true)); } - public void testRemoveInnerField() { - document.removeField("fizz.buzz"); - assertThat(document.getSourceAndMetadata().size(), equalTo(11)); - assertThat(document.getSourceAndMetadata().get("fizz"), instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map map = (Map) document.getSourceAndMetadata().get("fizz"); - assertThat(map.size(), equalTo(3)); - assertThat(map.containsKey("buzz"), equalTo(false)); - - document.removeField("fizz.foo_null"); - assertThat(map.size(), equalTo(2)); - assertThat(document.getSourceAndMetadata().size(), equalTo(11)); - assertThat(document.getSourceAndMetadata().containsKey("fizz"), equalTo(true)); + public void testRemoveInnerField() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.removeField("fizz.buzz"); + assertThat(doc.getSourceAndMetadata().size(), equalTo(15)); + assertThat(doc.getSourceAndMetadata().get("fizz"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map map = (Map) doc.getSourceAndMetadata().get("fizz"); + assertThat(map.size(), equalTo(3)); + assertThat(map.containsKey("buzz"), equalTo(false)); + + doc.removeField("fizz.foo_null"); + assertThat(map.size(), equalTo(2)); + assertThat(doc.getSourceAndMetadata().size(), equalTo(15)); + assertThat(doc.getSourceAndMetadata().containsKey("fizz"), equalTo(true)); + + doc.removeField("fizz.1"); + assertThat(map.size(), equalTo(1)); + assertThat(doc.getSourceAndMetadata().size(), equalTo(15)); + assertThat(doc.getSourceAndMetadata().containsKey("fizz"), equalTo(true)); + + doc.removeField("fizz.list"); + assertThat(map.size(), equalTo(0)); + assertThat(doc.getSourceAndMetadata().size(), equalTo(15)); + assertThat(doc.getSourceAndMetadata().containsKey("fizz"), equalTo(true)); + }); + doWithAccessPattern(FLEXIBLE, (doc) -> { + doc.removeField("dots.foo.bar.baz"); + assertThat(doc.getSourceAndMetadata().size(), equalTo(15)); + assertThat(doc.getSourceAndMetadata().get("dots"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map dots = (Map) doc.getSourceAndMetadata().get("dots"); + assertThat(dots.size(), equalTo(5)); + assertThat(dots.containsKey("foo.bar.baz"), equalTo(false)); - document.removeField("fizz.1"); - assertThat(map.size(), equalTo(1)); - assertThat(document.getSourceAndMetadata().size(), equalTo(11)); - assertThat(document.getSourceAndMetadata().containsKey("fizz"), equalTo(true)); + doc.removeField("dots.foo.bar.null"); + assertThat(dots.size(), equalTo(4)); + assertThat(dots.containsKey("foo.bar.null"), equalTo(false)); + assertThat(doc.getSourceAndMetadata().size(), equalTo(15)); + assertThat(doc.getSourceAndMetadata().containsKey("dots"), equalTo(true)); - document.removeField("fizz.list"); - assertThat(map.size(), equalTo(0)); - assertThat(document.getSourceAndMetadata().size(), equalTo(11)); - assertThat(document.getSourceAndMetadata().containsKey("fizz"), equalTo(true)); + doc.removeField("dots.arrays.dotted.strings"); + @SuppressWarnings("unchecked") + Map arrays = (Map) dots.get("arrays"); + assertThat(dots.size(), equalTo(4)); + assertThat(arrays.size(), equalTo(2)); + assertThat(arrays.containsKey("dotted.strings"), equalTo(false)); + assertThat(doc.getSourceAndMetadata().size(), equalTo(15)); + assertThat(doc.getSourceAndMetadata().containsKey("dots"), equalTo(true)); + }); } - public void testRemoveNonExistingField() { + public void testRemoveNonExistingField() throws Exception { try { - document.removeField("does_not_exist"); + doWithRandomAccessPattern((doc) -> doc.removeField("does_not_exist")); fail("remove field should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("field [does_not_exist] not present as part of path [does_not_exist]")); } + + try { + doWithAccessPattern(CLASSIC, (doc) -> doc.removeField("does.not.exist")); + fail("remove field should have failed"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("field [does] not present as part of path [does.not.exist]")); + } + + try { + doWithAccessPattern(FLEXIBLE, (doc) -> doc.removeField("does.not.exist")); + fail("remove field should have failed"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("field [does.not.exist] not present as part of path [does.not.exist]")); + } } - public void testRemoveExistingParentTypeMismatch() { + public void testRemoveExistingParentTypeMismatch() throws Exception { try { - document.removeField("foo.foo.bar"); + doWithRandomAccessPattern((doc) -> doc.removeField("foo.foo.bar")); fail("remove field should have failed"); } catch (IllegalArgumentException e) { assertThat( @@ -938,103 +1655,148 @@ public void testRemoveExistingParentTypeMismatch() { equalTo("cannot resolve [foo] from object of type [java.lang.String] as part of path [foo.foo.bar]") ); } + + try { + doWithAccessPattern(FLEXIBLE, (doc) -> doc.removeField("dots.foo.bar.baz.qux.quux")); + fail("remove field should have failed"); + } catch (IllegalArgumentException e) { + assertThat( + e.getMessage(), + equalTo("cannot resolve [qux] from object of type [java.lang.String] as part of path [dots.foo.bar.baz.qux.quux]") + ); + } } - public void testRemoveSourceObject() { + public void testRemoveSourceObject() throws Exception { try { - document.removeField("_source"); + doWithRandomAccessPattern((doc) -> doc.removeField("_source")); fail("remove field should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("field [_source] not present as part of path [_source]")); } } - public void testRemoveIngestObject() { - document.removeField("_ingest"); - assertThat(document.getSourceAndMetadata().size(), equalTo(10)); - assertThat(document.getSourceAndMetadata().containsKey("_ingest"), equalTo(false)); + public void testRemoveIngestObject() throws Exception { + doWithRandomAccessPattern((doc) -> { + doc.removeField("_ingest"); + assertThat(doc.getSourceAndMetadata().size(), equalTo(14)); + assertThat(doc.getSourceAndMetadata().containsKey("_ingest"), equalTo(false)); + }); } - public void testRemoveEmptyPathAfterStrippingOutPrefix() { + public void testRemoveEmptyPathAfterStrippingOutPrefix() throws Exception { try { - document.removeField("_source."); + doWithRandomAccessPattern((doc) -> doc.removeField("_source.")); fail("set field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path [_source.] is not valid")); } try { - document.removeField("_ingest."); + doWithRandomAccessPattern((doc) -> doc.removeField("_ingest.")); fail("set field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path [_ingest.] is not valid")); } } - public void testListRemoveField() { - document.removeField("list.0.field"); - assertThat(document.getSourceAndMetadata().size(), equalTo(11)); - assertThat(document.getSourceAndMetadata().containsKey("list"), equalTo(true)); - Object object = document.getSourceAndMetadata().get("list"); - assertThat(object, instanceOf(List.class)); - @SuppressWarnings("unchecked") - List list = (List) object; - assertThat(list.size(), equalTo(2)); - object = list.get(0); - assertThat(object, instanceOf(Map.class)); - @SuppressWarnings("unchecked") - Map map = (Map) object; - assertThat(map.size(), equalTo(0)); - document.removeField("list.0"); - assertThat(list.size(), equalTo(1)); - assertThat(list.get(0), nullValue()); + public void testListRemoveField() throws Exception { + doWithAccessPattern(CLASSIC, (doc) -> { + doc.removeField("list.0.field"); + assertThat(doc.getSourceAndMetadata().size(), equalTo(15)); + assertThat(doc.getSourceAndMetadata().containsKey("list"), equalTo(true)); + Object object = doc.getSourceAndMetadata().get("list"); + assertThat(object, instanceOf(List.class)); + @SuppressWarnings("unchecked") + List list = (List) object; + assertThat(list.size(), equalTo(2)); + object = list.get(0); + assertThat(object, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map map = (Map) object; + assertThat(map.size(), equalTo(0)); + document.removeField("list.0"); + assertThat(list.size(), equalTo(1)); + assertThat(list.get(0), nullValue()); + }); + // TODO: Flexible will have a new notation for list indexing - For now it does not support traversing lists + doWithAccessPattern(FLEXIBLE, (doc) -> { + var illegalArgument = expectThrows(IllegalArgumentException.class, () -> doc.removeField("list.0.field")); + assertThat(illegalArgument.getMessage(), equalTo("path [list.0.field] is not valid")); + }); } - public void testRemoveFieldValueNotFoundNullParent() { + public void testRemoveFieldValueNotFoundNullParent() throws Exception { try { - document.removeField("fizz.foo_null.not_there"); + doWithRandomAccessPattern((doc) -> doc.removeField("fizz.foo_null.not_there")); fail("get field value should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("cannot remove [not_there] from null as part of path [fizz.foo_null.not_there]")); } + try { + doWithAccessPattern(FLEXIBLE, (doc) -> doc.removeField("dots.foo.bar.null.not_there")); + fail("get field value should have failed"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("cannot remove [not_there] from null as part of path [dots.foo.bar.null.not_there]")); + } } - public void testNestedRemoveFieldTypeMismatch() { + public void testNestedRemoveFieldTypeMismatch() throws Exception { try { - document.removeField("fizz.1.bar"); + doWithRandomAccessPattern((doc) -> doc.removeField("fizz.1.bar")); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("cannot remove [bar] from object of type [java.lang.String] as part of path [fizz.1.bar]")); } + try { + doWithAccessPattern(FLEXIBLE, (doc) -> doc.removeField("dots.dotted.integers.a.bar.baz")); + } catch (IllegalArgumentException e) { + assertThat( + e.getMessage(), + equalTo("cannot resolve [bar] from object of type [java.lang.Integer] as part of path [dots.dotted.integers.a.bar.baz]") + ); + } } - public void testListRemoveFieldIndexNotNumeric() { + public void testListRemoveFieldIndexNotNumeric() throws Exception { try { - document.removeField("list.test"); + doWithAccessPattern(CLASSIC, (doc) -> doc.removeField("list.test")); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("[test] is not an integer, cannot be used as an index as part of path [list.test]")); } + // Flexible mode does not allow for interactions with arrays yet + try { + doWithAccessPattern(FLEXIBLE, (doc) -> doc.removeField("list.test")); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("path [list.test] is not valid")); + } } - public void testListRemoveFieldIndexOutOfBounds() { + public void testListRemoveFieldIndexOutOfBounds() throws Exception { try { - document.removeField("list.10"); + doWithAccessPattern(CLASSIC, (doc) -> doc.removeField("list.10")); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("[10] is out of bounds for array with length [2] as part of path [list.10]")); } + // Flexible mode does not allow for interactions with arrays yet + try { + doWithAccessPattern(FLEXIBLE, (doc) -> doc.removeField("list.10")); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("path [list.10] is not valid")); + } } - public void testRemoveNullField() { + public void testRemoveNullField() throws Exception { try { - document.removeField(null); + doWithRandomAccessPattern((doc) -> doc.removeField(null)); fail("remove field should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path cannot be null nor empty")); } } - public void testRemoveEmptyField() { + public void testRemoveEmptyField() throws Exception { try { - document.removeField(""); + doWithRandomAccessPattern((doc) -> doc.removeField("")); fail("remove field should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("path cannot be null nor empty")); @@ -1182,10 +1944,11 @@ public void testGetAllFields() { source.put("_id", "a123"); source.put("name", "eric clapton"); source.put("address", address); + source.put("name.display", "Eric Clapton"); Set result = IngestDocument.getAllFields(source); - assertThat(result, containsInAnyOrder("_id", "name", "address", "address.street", "address.number")); + assertThat(result, containsInAnyOrder("_id", "name", "address", "address.street", "address.number", "name.display")); } public void testIsMetadata() {