-
Notifications
You must be signed in to change notification settings - Fork 25.5k
Making changes to BulkRequest to enable component template substitutions in the simulate ingest API #112957
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Making changes to BulkRequest to enable component template substitutions in the simulate ingest API #112957
Changes from 4 commits
bfd61e9
d4f858b
5ecd044
8548354
9804dd8
2d0b183
27b5395
54f55e1
5f1e5ba
d5d1ee7
9b771f1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
pr: 112957 | ||
summary: Making changes to `BulkRequest` to enable component template substitutions | ||
in the simulate ingest API | ||
area: Ingest Node | ||
type: enhancement | ||
issues: [] |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -9,16 +9,24 @@ | |
|
||
package org.elasticsearch.action.bulk; | ||
|
||
import org.elasticsearch.TransportVersions; | ||
import org.elasticsearch.cluster.metadata.AliasMetadata; | ||
import org.elasticsearch.cluster.metadata.ComponentTemplate; | ||
import org.elasticsearch.cluster.metadata.DataStreamLifecycle; | ||
import org.elasticsearch.cluster.metadata.Template; | ||
import org.elasticsearch.common.compress.CompressedXContent; | ||
import org.elasticsearch.common.io.stream.StreamInput; | ||
import org.elasticsearch.common.io.stream.StreamOutput; | ||
import org.elasticsearch.common.settings.Settings; | ||
import org.elasticsearch.core.Nullable; | ||
|
||
import java.io.IOException; | ||
import java.util.HashMap; | ||
import java.util.Map; | ||
|
||
/** | ||
* This extends BulkRequest with support for providing substitute pipeline definitions. In a user request, the pipeline substitutions | ||
* will look something like this: | ||
* This extends BulkRequest with support for providing substitute pipeline definitions and component template definitions. In a user | ||
* request, the substitutions will look something like this: | ||
* | ||
* "pipeline_substitutions": { | ||
* "my-pipeline-1": { | ||
|
@@ -45,6 +53,27 @@ | |
* } | ||
* ] | ||
* } | ||
* }, | ||
* "component_template_substitutions": { | ||
* "my-template-1": { | ||
* "settings": { | ||
* "number_of_shards": 1 | ||
* }, | ||
* "mappings": { | ||
* "_source": { | ||
* "enabled": false | ||
* }, | ||
* "properties": { | ||
* "host_name": { | ||
* "type": "keyword" | ||
* }, | ||
* "created_at": { | ||
* "type": "date", | ||
* "format": "EEE MMM dd HH:mm:ss Z yyyy" | ||
* } | ||
* } | ||
* } | ||
* } | ||
* } | ||
* | ||
* The pipelineSubstitutions Map held by this class is intended to be the result of XContentHelper.convertToMap(). The top-level keys | ||
|
@@ -53,27 +82,42 @@ | |
*/ | ||
public class SimulateBulkRequest extends BulkRequest { | ||
private final Map<String, Map<String, Object>> pipelineSubstitutions; | ||
private final Map<String, Map<String, Object>> componentTemplateSubstitutions; | ||
|
||
/** | ||
* @param pipelineSubstitutions The pipeline definitions that are to be used in place of any pre-existing pipeline definitions with | ||
* the same pipelineId. The key of the map is the pipelineId, and the value the pipeline definition as | ||
* parsed by XContentHelper.convertToMap(). | ||
* @param componentTemplateSubstitutions The component template definitions that are to be used in place of any pre-existing | ||
* component template definitions with the same name. | ||
*/ | ||
public SimulateBulkRequest(@Nullable Map<String, Map<String, Object>> pipelineSubstitutions) { | ||
public SimulateBulkRequest( | ||
@Nullable Map<String, Map<String, Object>> pipelineSubstitutions, | ||
@Nullable Map<String, Map<String, Object>> componentTemplateSubstitutions | ||
) { | ||
super(); | ||
this.pipelineSubstitutions = pipelineSubstitutions; | ||
this.componentTemplateSubstitutions = componentTemplateSubstitutions; | ||
} | ||
|
||
@SuppressWarnings("unchecked") | ||
public SimulateBulkRequest(StreamInput in) throws IOException { | ||
super(in); | ||
this.pipelineSubstitutions = (Map<String, Map<String, Object>>) in.readGenericValue(); | ||
if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_COMPONENT_TEMPLATES_SUBSTITUTIONS)) { | ||
this.componentTemplateSubstitutions = (Map<String, Map<String, Object>>) in.readGenericValue(); | ||
} else { | ||
componentTemplateSubstitutions = Map.of(); | ||
} | ||
} | ||
|
||
@Override | ||
public void writeTo(StreamOutput out) throws IOException { | ||
super.writeTo(out); | ||
out.writeGenericValue(pipelineSubstitutions); | ||
if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_COMPONENT_TEMPLATES_SUBSTITUTIONS)) { | ||
out.writeGenericValue(componentTemplateSubstitutions); | ||
} | ||
} | ||
|
||
public Map<String, Map<String, Object>> getPipelineSubstitutions() { | ||
|
@@ -84,4 +128,41 @@ public Map<String, Map<String, Object>> getPipelineSubstitutions() { | |
public boolean isSimulated() { | ||
return true; | ||
} | ||
|
||
@Override | ||
public Map<String, ComponentTemplate> getComponentTemplateSubstitutions() throws IOException { | ||
if (componentTemplateSubstitutions == null) { | ||
return Map.of(); | ||
} | ||
Map<String, ComponentTemplate> result = new HashMap<>(componentTemplateSubstitutions.size()); | ||
for (Map.Entry<String, Map<String, Object>> rawEntry : componentTemplateSubstitutions.entrySet()) { | ||
result.put(rawEntry.getKey(), convertRawTemplateToComponentTemplate(rawEntry.getValue())); | ||
} | ||
return result; | ||
} | ||
|
||
@SuppressWarnings("unchecked") | ||
private static ComponentTemplate convertRawTemplateToComponentTemplate(Map<String, Object> rawTemplate) throws IOException { | ||
Settings settings = null; | ||
CompressedXContent mappings = null; | ||
if (rawTemplate.containsKey("mappings")) { | ||
mappings = new CompressedXContent((Map<String, Object>) rawTemplate.get("mappings")); | ||
} | ||
if (rawTemplate.containsKey("settings")) { | ||
settings = Settings.builder().loadFromMap((Map<String, ?>) rawTemplate.get("settings")).build(); | ||
} | ||
Map<String, AliasMetadata> aliases = null; | ||
DataStreamLifecycle lifecycle = null; | ||
|
||
Template template = new Template(settings, mappings, aliases, lifecycle); | ||
return new ComponentTemplate(template, null, null); | ||
} | ||
|
||
@Override | ||
public BulkRequest shallowClone() { | ||
BulkRequest bulkRequest = new SimulateBulkRequest(pipelineSubstitutions, componentTemplateSubstitutions); | ||
bulkRequest.setRefreshPolicy(getRefreshPolicy()); | ||
bulkRequest.waitForActiveShards(waitForActiveShards()); | ||
bulkRequest.timeout(timeout()); | ||
return bulkRequest; | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
BulkRequest
has many other things than only the refresh policy, active shards, and timeout. Are we going to copy the rest of them also? Should the javadoc reflect what's being copied?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I went his way because this is what BulkRequestModifier did (this code was moved out of there). I wavered on whether I ought to add the other fields or not, and wound up just repeating and testing the previous behavior. Any thoughts?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think just documenting it in the javadoc is fine
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I wound up adding the rest of the fields just in case (I don't think they'd intentionally been left out -- I think they were just added after the BulkRequestModifier method had been written).