Optimized field visitor for ES|QL loading from _ignored_source #132428
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the "Elastic License | ||
| * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side | ||
| * Public License v 1"; you may not use this file except in compliance with, at | ||
| * your election, the "Elastic License 2.0", the "GNU Affero General Public | ||
| * License v3.0 only", or the "Server Side Public License, v 1". | ||
| */ | ||
|
|
||
| package org.elasticsearch.index.fieldvisitor; | ||
|
|
||
| import org.apache.lucene.index.FieldInfo; | ||
| import org.apache.lucene.index.LeafReaderContext; | ||
| import org.apache.lucene.index.StoredFieldVisitor; | ||
| import org.apache.lucene.util.BytesRef; | ||
| import org.elasticsearch.common.bytes.BytesReference; | ||
| import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; | ||
| import org.elasticsearch.search.fetch.StoredFieldsSpec; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.HashMap; | ||
| import java.util.HashSet; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Set; | ||
|
|
||
| class IgnoredSourceFieldLoader extends StoredFieldLoader { | ||
| private final boolean forceSequentialReader; | ||
| private final Map<String, Set<String>> potentialFieldsInIgnoreSource; | ||
| private final Set<String> fieldNames; | ||
|
|
||
| IgnoredSourceFieldLoader(StoredFieldsSpec spec, boolean forceSequentialReader) { | ||
| assert IgnoredSourceFieldLoader.supports(spec); | ||
|
|
||
| fieldNames = new HashSet<>(spec.ignoredFieldsSpec().requiredIgnoredFields()); | ||
| this.forceSequentialReader = forceSequentialReader; | ||
|
|
||
| HashMap<String, Set<String>> potentialFieldsInIgnoreSource = new HashMap<>(); | ||
| for (String requiredIgnoredField : spec.ignoredFieldsSpec().requiredIgnoredFields()) { | ||
| for (String potentialStoredField : spec.ignoredFieldsSpec().format().requiredStoredFields(requiredIgnoredField)) { | ||
| potentialFieldsInIgnoreSource.computeIfAbsent(potentialStoredField, k -> new HashSet<>()).add(requiredIgnoredField); | ||
| } | ||
| } | ||
| this.potentialFieldsInIgnoreSource = potentialFieldsInIgnoreSource; | ||
| } | ||
|
|
||
| @Override | ||
| public LeafStoredFieldLoader getLoader(LeafReaderContext ctx, int[] docs) throws IOException { | ||
| var reader = forceSequentialReader ? sequentialReader(ctx) : reader(ctx, docs); | ||
| var visitor = new SFV(fieldNames, potentialFieldsInIgnoreSource); | ||
| return new LeafStoredFieldLoader() { | ||
|
|
||
| private int doc = -1; | ||
|
|
||
| @Override | ||
| public void advanceTo(int doc) throws IOException { | ||
| if (doc != this.doc) { | ||
| visitor.reset(); | ||
| reader.accept(doc, visitor); | ||
| this.doc = doc; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public BytesReference source() { | ||
| assert false : "source() is not supported by IgnoredSourceFieldLoader"; | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| assert false : "id() is not supported by IgnoredSourceFieldLoader"; | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public String routing() { | ||
| assert false : "routing() is not supported by IgnoredSourceFieldLoader"; | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public Map<String, List<Object>> storedFields() { | ||
| return visitor.values; | ||
| } | ||
| }; | ||
| } | ||
|
|
||
| @Override | ||
| public List<String> fieldsToLoad() { | ||
| return potentialFieldsInIgnoreSource.keySet().stream().toList(); | ||
| } | ||
|
|
||
| static class SFV extends StoredFieldVisitor { | ||
|
There was a problem hiding this comment. In the future we can perhaps explore an implementation that is tuned for just fetching one stored field. |
||
| final Map<String, List<Object>> values = new HashMap<>(); | ||
| final Set<String> fieldNames; | ||
| private final Set<String> unvisitedFields; | ||
| final Map<String, Set<String>> potentialFieldsInIgnoreSource; | ||
|
|
||
| SFV(Set<String> fieldNames, Map<String, Set<String>> potentialFieldsInIgnoreSource) { | ||
| this.fieldNames = fieldNames; | ||
| this.unvisitedFields = new HashSet<>(fieldNames); | ||
| this.potentialFieldsInIgnoreSource = potentialFieldsInIgnoreSource; | ||
| } | ||
|
|
||
| @Override | ||
| public Status needsField(FieldInfo fieldInfo) throws IOException { | ||
| if (unvisitedFields.isEmpty()) { | ||
| return Status.STOP; | ||
| } | ||
|
|
||
| Set<String> foundFields = potentialFieldsInIgnoreSource.get(fieldInfo.name); | ||
| if (foundFields == null) { | ||
| return Status.NO; | ||
| } | ||
|
|
||
| unvisitedFields.removeAll(foundFields); | ||
| return Status.YES; | ||
| } | ||
|
|
||
| @Override | ||
| public void binaryField(FieldInfo fieldInfo, byte[] value) { | ||
| values.computeIfAbsent(fieldInfo.name, k -> new ArrayList<>()).add(new BytesRef(value)); | ||
| } | ||
|
|
||
| void reset() { | ||
| values.clear(); | ||
| unvisitedFields.addAll(fieldNames); | ||
| } | ||
|
|
||
| } | ||
|
|
||
| static boolean supports(StoredFieldsSpec spec) { | ||
| return spec.onlyRequiresIgnoredFields() | ||
| && spec.ignoredFieldsSpec().format() == IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE; | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the "Elastic License | ||
| * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side | ||
| * Public License v 1"; you may not use this file except in compliance with, at | ||
| * your election, the "Elastic License 2.0", the "GNU Affero General Public | ||
| * License v3.0 only", or the "Server Side Public License, v 1". | ||
| */ | ||
|
|
||
| package org.elasticsearch.index.mapper; | ||
|
|
||
| import org.elasticsearch.ElasticsearchException; | ||
|
|
||
| import java.util.HashSet; | ||
| import java.util.Set; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| /** | ||
| * Defines which fields need to be loaded from _ignored_source during a fetch. | ||
| */ | ||
| public record IgnoredFieldsSpec(Set<String> requiredIgnoredFields, IgnoredSourceFieldMapper.IgnoredSourceFormat format) { | ||
| public static IgnoredFieldsSpec NONE = new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE); | ||
|
|
||
| public boolean noRequirements() { | ||
| return requiredIgnoredFields.isEmpty(); | ||
| } | ||
|
|
||
| public IgnoredFieldsSpec merge(IgnoredFieldsSpec other) { | ||
| if (this.format == IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE) { | ||
| return other; | ||
| } | ||
| if (other.format == IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE) { | ||
| return this; | ||
| } | ||
| if (other.requiredIgnoredFields.isEmpty()) { | ||
| return this; | ||
| } | ||
| if (this.requiredIgnoredFields.isEmpty()) { | ||
| return other; | ||
| } | ||
|
|
||
| if (this.format != other.format) { | ||
| throw new ElasticsearchException( | ||
| "failed to merge IgnoredFieldsSpec with differing formats " + this.format.name() + "," + other.format.name() | ||
| ); | ||
| } | ||
|
|
||
| Set<String> mergedFields = new HashSet<>(requiredIgnoredFields); | ||
| mergedFields.addAll(other.requiredIgnoredFields); | ||
| return new IgnoredFieldsSpec(mergedFields, format); | ||
| } | ||
|
|
||
| /** | ||
| * Get the set of stored fields required to load the specified fields from _ignored_source. | ||
| */ | ||
| public Set<String> requiredStoredFields() { | ||
| return requiredIgnoredFields.stream().flatMap(field -> format.requiredStoredFields(field).stream()).collect(Collectors.toSet()); | ||
|
|
||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I hoping that we can optimize/simplify in a follow up change. This logic is now required for fields with dots and then we don't know which field part there is actually a stored field for.
Ideally we would know based from the mapping what stored field we need. For example if
attributes.host.ipis requested and noattributesfield is mapped, then the stored field should be_ignored_source.attributes. If attributes is mapped, but host isn't then the required stored field to load would be_ignored_source.attributes.host.I also think
requiredStoredFields(...)also always include the_docfield (viaFallbackSyntheticSourceBlockLoader.splitIntoFieldPaths(...))?Would be great if we need less maps and no sets in
SFV.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good point, this can probably be optimized by using the mapping to figure out which stored field contains the value instead of just looking at all possible parent stored fields. But let's leave that as a follow-up.