diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java index b75b144ed616d..6a5d437f75d81 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java @@ -8,6 +8,7 @@ */ package org.elasticsearch.logstashbridge.ingest; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.ingest.Pipeline; import org.elasticsearch.logstashbridge.StableBridgeAPI; import org.elasticsearch.logstashbridge.script.ScriptServiceBridge; @@ -20,6 +21,7 @@ public static PipelineBridge wrap(final Pipeline pipeline) { return new PipelineBridge(pipeline); } + @FixForMultiProject(description = "should we pass a non-null project ID here?") public static PipelineBridge create( String id, Map config, @@ -27,7 +29,13 @@ public static PipelineBridge create( ScriptServiceBridge scriptServiceBridge ) throws Exception { return wrap( - Pipeline.create(id, config, StableBridgeAPI.unwrap(processorFactories), StableBridgeAPI.unwrapNullable(scriptServiceBridge)) + Pipeline.create( + id, + config, + StableBridgeAPI.unwrap(processorFactories), + StableBridgeAPI.unwrapNullable(scriptServiceBridge), + null + ) ); } diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java index 577865f55c752..ecf9ab44f55b8 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java @@ -8,6 +8,7 @@ */ package org.elasticsearch.logstashbridge.ingest; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.TimeValue; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.ingest.Processor; @@ -118,7 +119,7 @@ static Factory wrap(final Processor.Factory delegate) { @Override default Processor.Factory unwrap() { final Factory stableAPIFactory = this; - return (registry, tag, description, config) -> stableAPIFactory.create( + return (registry, tag, description, config, projectId) -> stableAPIFactory.create( StableBridgeAPI.wrap(registry, Factory::wrap), tag, description, @@ -131,6 +132,7 @@ private Wrapped(final Processor.Factory delegate) { super(delegate); } + @FixForMultiProject(description = "should we pass a non-null project ID here?") @Override public ProcessorBridge create( final Map registry, @@ -138,7 +140,9 @@ public ProcessorBridge create( final String description, final Map config ) throws Exception { - return ProcessorBridge.wrap(this.delegate.create(StableBridgeAPI.unwrap(registry), processorTag, description, config)); + return ProcessorBridge.wrap( + this.delegate.create(StableBridgeAPI.unwrap(registry), processorTag, description, config, null) + ); } @Override diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java index 630ec8d8fe63c..4bb51a4bf0dfc 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java @@ -403,9 +403,9 @@ public Map getProcessors(Processor.Parameters paramet Map processors = new HashMap<>(); processors.put( "drop", - (factories, tag, description, config) -> new TestProcessor(tag, "drop", description, ingestDocument -> null) + (factories, tag, description, config, projectId) -> new TestProcessor(tag, "drop", description, ingestDocument -> null) ); - processors.put("reroute", (factories, tag, description, config) -> { + processors.put("reroute", (factories, tag, description, config, projectId) -> { String destination = (String) config.remove("destination"); return new TestProcessor( tag, @@ -416,7 +416,12 @@ public Map getProcessors(Processor.Parameters paramet }); processors.put( "fail", - (processorFactories, tag, description, config) -> new TestProcessor(tag, "fail", description, new RuntimeException()) + (processorFactories, tag, description, config, projectId) -> new TestProcessor( + tag, + "fail", + description, + new RuntimeException() + ) ); return processors; } diff --git a/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java index 83a7bdf7e224a..863155f07f883 100644 --- a/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java +++ b/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java @@ -15,6 +15,7 @@ import org.apache.tika.metadata.Office; import org.apache.tika.metadata.TikaCoreProperties; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; @@ -232,7 +233,8 @@ public AttachmentProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) { String field = readStringProperty(TYPE, processorTag, config, "field"); String resourceName = readOptionalStringProperty(TYPE, processorTag, config, "resource_name"); diff --git a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java index 02d8b71afb856..5fa22d9003c7d 100644 --- a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java +++ b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java @@ -36,7 +36,7 @@ public void testBuildDefaults() throws Exception { String processorTag = randomAlphaOfLength(10); - AttachmentProcessor processor = factory.create(null, processorTag, null, config); + AttachmentProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("attachment")); @@ -57,7 +57,7 @@ public void testConfigureIndexedChars() throws Exception { config.put("indexed_chars", indexedChars); String processorTag = randomAlphaOfLength(10); - AttachmentProcessor processor = factory.create(null, processorTag, null, config); + AttachmentProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getIndexedChars(), is(indexedChars)); assertFalse(processor.isIgnoreMissing()); @@ -73,7 +73,7 @@ public void testBuildTargetField() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("target_field", "_field"); - AttachmentProcessor processor = factory.create(null, null, null, config); + AttachmentProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("_field")); assertFalse(processor.isIgnoreMissing()); @@ -97,7 +97,7 @@ public void testBuildFields() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("properties", fieldNames); - AttachmentProcessor processor = factory.create(null, null, null, config); + AttachmentProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(properties)); assertFalse(processor.isIgnoreMissing()); @@ -114,7 +114,7 @@ public void testBuildIllegalFieldOption() throws Exception { config.put("field", "_field"); config.put("properties", Collections.singletonList("invalid")); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("exception expected"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), containsString("[properties] illegal field option [invalid]")); @@ -128,7 +128,7 @@ public void testBuildIllegalFieldOption() throws Exception { config.put("field", "_field"); config.put("properties", "invalid"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("exception expected"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]")); @@ -148,7 +148,7 @@ public void testIgnoreMissing() throws Exception { String processorTag = randomAlphaOfLength(10); - AttachmentProcessor processor = factory.create(null, processorTag, null, config); + AttachmentProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("attachment")); @@ -169,7 +169,7 @@ public void testRemoveBinary() throws Exception { String processorTag = randomAlphaOfLength(10); - AttachmentProcessor processor = factory.create(null, processorTag, null, config); + AttachmentProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("attachment")); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java index 695666a22a7c4..10397c18fc543 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -108,7 +109,8 @@ public AbstractStringProcessor create( Map registry, String tag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(processorType, tag, config, "field"); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(processorType, tag, config, "ignore_missing", false); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java index 0992983dd7d94..a812e6eef550a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -73,7 +74,8 @@ public AppendProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value"); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java index ab158acb60fdf..cda21f34b575f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; @@ -297,7 +298,8 @@ public CommunityIdProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP); String sourcePortField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_port", DEFAULT_SOURCE_PORT); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java index 5c72d1e444185..93ccb020d68b4 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; @@ -216,7 +217,8 @@ public ConvertProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type"); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java index d7abe08a7bfc5..49af5aa8c9530 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -93,7 +94,8 @@ public CsvProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String quote = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "quote", "\""); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java index a7da324899488..d47e1a9d7d311 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java @@ -10,6 +10,7 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; @@ -150,7 +151,8 @@ public DateIndexNameProcessor create( Map registry, String tag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale"); String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone"); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java index 22db5a330fb45..05951d895b6d4 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java @@ -10,6 +10,7 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.util.LocaleUtils; @@ -183,7 +184,8 @@ public DateProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java index d2d2f155dfa4d..fd5113a6e2d86 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.dissect.DissectParser; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; @@ -60,7 +61,8 @@ public DissectProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String pattern = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "pattern"); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java index 93dc45bf67e6d..910b917856417 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -120,7 +121,8 @@ public Processor create( Map processorFactories, String tag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); if (field.contains(".") == false && field.equals("*") == false) { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java index f09e1b3fc58c7..e0cf74639d594 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -60,7 +61,8 @@ public FailProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String message = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "message"); TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate( diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java index ed4d0b21a9119..4fbc71f64398e 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.Strings; import org.elasticsearch.common.hash.Murmur3Hasher; import org.elasticsearch.common.util.ByteUtils; @@ -227,7 +228,8 @@ public FingerprintProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { List fields = ConfigurationUtils.readList(TYPE, processorTag, config, "fields"); if (fields.size() < 1) { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java index 7831e743a712e..91a2ccbd66801 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.Maps; import org.elasticsearch.ingest.AbstractProcessor; @@ -235,8 +236,13 @@ public static final class Factory implements Processor.Factory { } @Override - public ForEachProcessor create(Map factories, String tag, String description, Map config) - throws Exception { + public ForEachProcessor create( + Map factories, + String tag, + String description, + Map config, + ProjectId projectId + ) throws Exception { String field = readStringProperty(TYPE, tag, config, "field"); boolean ignoreMissing = readBooleanProperty(TYPE, tag, config, "ignore_missing", false); Map> processorConfig = readMap(TYPE, tag, config, "processor"); @@ -245,7 +251,7 @@ public ForEachProcessor create(Map factories, String throw newConfigurationException(TYPE, tag, "processor", "Must specify exactly one processor type"); } Map.Entry> entry = entries.iterator().next(); - Processor processor = ConfigurationUtils.readProcessor(factories, scriptService, entry.getKey(), entry.getValue()); + Processor processor = ConfigurationUtils.readProcessor(factories, scriptService, entry.getKey(), entry.getValue(), projectId); return new ForEachProcessor(tag, description, field, processor, ignoreMissing); } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java index a7bcad63d3001..dd8d6ec202558 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.grok.Grok; import org.elasticsearch.grok.GrokBuiltinPatterns; import org.elasticsearch.grok.MatcherWatchdog; @@ -150,7 +151,8 @@ public GrokProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); List matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns"); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java index 8c7ca42843842..fcc0bd3ca240e 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -71,7 +72,8 @@ public JoinProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String separator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java index 69c5699912f5d..47e7d5846d592 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.core.Strings; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; @@ -217,7 +218,8 @@ public JsonProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String targetField = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "target_field"); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java index e51457adef3d6..902f282c788ed 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.core.Predicates; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; @@ -256,7 +257,8 @@ public KeyValueProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); TemplateScript.Factory fieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java index 9551613b1e911..7257eaf71a3cb 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.network.CIDRUtils; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.ingest.AbstractProcessor; @@ -251,7 +252,8 @@ public NetworkDirectionProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { final String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP); final String destIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "destination_ip", DEFAULT_DEST_IP); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java index 3b212c71f2220..fa6df0c7b4f78 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java @@ -11,6 +11,7 @@ import org.apache.http.conn.util.PublicSuffixMatcher; import org.apache.http.conn.util.PublicSuffixMatcherLoader; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.Strings; import org.elasticsearch.core.Nullable; import org.elasticsearch.ingest.AbstractProcessor; @@ -140,7 +141,8 @@ public RegisteredDomainProcessor create( Map registry, String tag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, "target_field", DEFAULT_TARGET_FIELD); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java index e92fd5767df38..1b1dc67dcb8d9 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -109,7 +110,8 @@ public RemoveProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { final List compiledTemplatesToRemove = getTemplates(processorTag, config, "field"); final List compiledTemplatesToKeep = getTemplates(processorTag, config, "keep"); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java index ce1783fa18a00..61bf831de5965 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -110,7 +111,8 @@ public RenameProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); TemplateScript.Factory fieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java index 57479749e1794..6580a5af3d005 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.core.Nullable; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; @@ -175,7 +176,8 @@ public RerouteProcessor create( Map processorFactories, String tag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { List type; try { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java index a2798dbefd0d9..83e4c1b4bfe82 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; @@ -105,7 +106,8 @@ public ScriptProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { try ( XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent).map(config); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java index 0ffef98580c4a..c7c9f6ac81843 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -109,7 +110,8 @@ public SetProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String copyFrom = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "copy_from"); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java index 0614fa90b67f0..a0ab0d8ffe2bf 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -115,7 +116,8 @@ public SortProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, FIELD); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java index d46652f7a5a79..f0800fa35e047 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -99,7 +100,8 @@ public SplitProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java index 5b6144ba8eab9..d2441e4de7ed8 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; @@ -45,7 +46,8 @@ public Processor create( Map processorFactories, String tag, String description, - Map config + Map config, + ProjectId projectId ) { return new TerminateProcessor(tag, description); } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java index 40116ec99abbd..e1383c1dc89bb 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; @@ -184,7 +185,8 @@ public UriPartsProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", "url"); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java index 4bdeba65e4638..aa6ca4df19240 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java @@ -37,7 +37,7 @@ public void testCreate() throws Exception { Map config = new HashMap<>(); config.put("field", fieldName); - AbstractStringProcessor processor = factory.create(null, processorTag, null, modifyConfig(config)); + AbstractStringProcessor processor = factory.create(null, processorTag, null, modifyConfig(config), null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(fieldName)); assertThat(processor.isIgnoreMissing(), is(false)); @@ -54,7 +54,7 @@ public void testCreateWithIgnoreMissing() throws Exception { config.put("field", fieldName); config.put("ignore_missing", true); - AbstractStringProcessor processor = factory.create(null, processorTag, null, modifyConfig(config)); + AbstractStringProcessor processor = factory.create(null, processorTag, null, modifyConfig(config), null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(fieldName)); assertThat(processor.isIgnoreMissing(), is(true)); @@ -72,7 +72,7 @@ public void testCreateWithTargetField() throws Exception { config.put("field", fieldName); config.put("target_field", targetFieldName); - AbstractStringProcessor processor = factory.create(null, processorTag, null, modifyConfig(config)); + AbstractStringProcessor processor = factory.create(null, processorTag, null, modifyConfig(config), null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(fieldName)); assertThat(processor.isIgnoreMissing(), is(false)); @@ -84,7 +84,7 @@ public void testCreateMissingField() throws Exception { AbstractStringProcessor.Factory factory = newFactory(); Map config = new HashMap<>(); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java index 790d9860de11e..565af0584efe6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java @@ -43,7 +43,7 @@ public void testCreate() throws Exception { } config.put("value", value); String processorTag = randomAlphaOfLength(10); - AppendProcessor appendProcessor = factory.create(null, processorTag, null, config); + AppendProcessor appendProcessor = factory.create(null, processorTag, null, config, null); assertThat(appendProcessor.getTag(), equalTo(processorTag)); assertThat(appendProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1")); assertThat(appendProcessor.getValue().copyAndResolve(Map.of()), equalTo(value)); @@ -53,7 +53,7 @@ public void testCreateNoFieldPresent() throws Exception { Map config = new HashMap<>(); config.put("value", "value1"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -64,7 +64,7 @@ public void testCreateNoValuePresent() throws Exception { Map config = new HashMap<>(); config.put("field", "field1"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[value] required property is missing")); @@ -76,7 +76,7 @@ public void testCreateNullValue() throws Exception { config.put("field", "field1"); config.put("value", null); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[value] required property is missing")); @@ -91,7 +91,7 @@ public void testInvalidMustacheTemplate() throws Exception { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows( ElasticsearchException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); @@ -105,7 +105,7 @@ public void testMediaType() throws Exception { config.put("value", "value1"); config.put("media_type", expectedMediaType); String processorTag = randomAlphaOfLength(10); - AppendProcessor appendProcessor = factory.create(null, processorTag, null, config); + AppendProcessor appendProcessor = factory.create(null, processorTag, null, config, null); assertThat(appendProcessor.getTag(), equalTo(processorTag)); // invalid media type @@ -117,7 +117,10 @@ public void testMediaType() throws Exception { config2.put("field", "field1"); config2.put("value", "value1"); config2.put("media_type", expectedMediaType); - ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config2)); + ElasticsearchException e = expectThrows( + ElasticsearchException.class, + () -> factory.create(null, processorTag, null, config2, null) + ); assertThat(e.getMessage(), containsString("property does not contain a supported media type [" + expectedMediaType + "]")); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java index 53fce7e2c0746..3426e9894c2b1 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java @@ -65,7 +65,7 @@ public void testCreate() throws Exception { config.put("ignore_missing", ignoreMissing); String processorTag = randomAlphaOfLength(10); - CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config); + CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config, null); assertThat(communityIdProcessor.getTag(), equalTo(processorTag)); assertThat(communityIdProcessor.getSourceIpField(), equalTo(sourceIpField)); assertThat(communityIdProcessor.getSourcePortField(), equalTo(sourcePortField)); @@ -87,27 +87,27 @@ public void testSeed() throws Exception { // negative seeds are rejected int tooSmallSeed = randomIntBetween(Integer.MIN_VALUE, -1); config.put("seed", Integer.toString(tooSmallSeed)); - ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); + ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null)); assertThat(e.getMessage(), containsString("must be a value between 0 and 65535")); // seeds >= 2^16 are rejected int tooBigSeed = randomIntBetween(65536, Integer.MAX_VALUE); config.put("seed", Integer.toString(tooBigSeed)); - e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); + e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null)); assertThat(e.getMessage(), containsString("must be a value between 0 and 65535")); // seeds between 0 and 2^16-1 are accepted int justRightSeed = randomIntBetween(0, 65535); byte[] expectedSeed = new byte[] { (byte) (justRightSeed >> 8), (byte) justRightSeed }; config.put("seed", Integer.toString(justRightSeed)); - CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config); + CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config, null); assertThat(communityIdProcessor.getSeed(), equalTo(expectedSeed)); } public void testRequiredFields() throws Exception { HashMap config = new HashMap<>(); String processorTag = randomAlphaOfLength(10); - CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config); + CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config, null); assertThat(communityIdProcessor.getTag(), equalTo(processorTag)); assertThat(communityIdProcessor.getSourceIpField(), equalTo(DEFAULT_SOURCE_IP)); assertThat(communityIdProcessor.getSourcePortField(), equalTo(DEFAULT_SOURCE_PORT)); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java index ad07526518173..492367b14b7c8 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java @@ -28,7 +28,7 @@ public void testCreate() throws Exception { config.put("field", "field1"); config.put("type", type.toString()); String processorTag = randomAlphaOfLength(10); - ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config); + ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config, null); assertThat(convertProcessor.getTag(), equalTo(processorTag)); assertThat(convertProcessor.getField(), equalTo("field1")); assertThat(convertProcessor.getTargetField(), equalTo("field1")); @@ -43,7 +43,7 @@ public void testCreateUnsupportedType() throws Exception { config.put("field", "field1"); config.put("type", type); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[type] type [" + type + "] not supported, cannot convert field.")); @@ -59,7 +59,7 @@ public void testCreateNoFieldPresent() throws Exception { String type = "type-" + randomAlphaOfLengthBetween(1, 10); config.put("type", type); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -71,7 +71,7 @@ public void testCreateNoTypePresent() throws Exception { Map config = new HashMap<>(); config.put("field", "field1"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[type] required property is missing")); @@ -86,7 +86,7 @@ public void testCreateWithExplicitTargetField() throws Exception { config.put("target_field", "field2"); config.put("type", type.toString()); String processorTag = randomAlphaOfLength(10); - ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config); + ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config, null); assertThat(convertProcessor.getTag(), equalTo(processorTag)); assertThat(convertProcessor.getField(), equalTo("field1")); assertThat(convertProcessor.getTargetField(), equalTo("field2")); @@ -102,7 +102,7 @@ public void testCreateWithIgnoreMissing() throws Exception { config.put("type", type.toString()); config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config); + ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config, null); assertThat(convertProcessor.getTag(), equalTo(processorTag)); assertThat(convertProcessor.getField(), equalTo("field1")); assertThat(convertProcessor.getTargetField(), equalTo("field1")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java index 22f69f88b7e1e..493496dac7bc0 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java @@ -31,7 +31,7 @@ public void testProcessorIsCreated() { properties.put("empty_value", "empty"); properties.put("trim", true); properties.put("ignore_missing", true); - CsvProcessor csv = factory.create(null, "csv", null, properties); + CsvProcessor csv = factory.create(null, "csv", null, properties, null); assertThat(csv, notNullValue()); assertThat(csv.field, equalTo("field")); assertThat(csv.headers, equalTo(new String[] { "target" })); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java index 1e558eaeb6fbb..fe8814b6c6ffc 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java @@ -28,7 +28,7 @@ public void testDefaults() throws Exception { config.put("field", "_field"); config.put("date_rounding", "y"); - DateIndexNameProcessor processor = factory.create(null, null, null, config); + DateIndexNameProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getDateFormats().size(), equalTo(1)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getIndexNamePrefixTemplate().newInstance(Map.of()).execute(), equalTo("")); @@ -45,7 +45,7 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("date_rounding", "y"); config.put("date_formats", List.of("UNIX", "UNIX_MS")); - DateIndexNameProcessor processor = factory.create(null, null, null, config); + DateIndexNameProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getDateFormats().size(), equalTo(2)); config = new HashMap<>(); @@ -54,7 +54,7 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("date_rounding", "y"); config.put("index_name_format", "yyyyMMdd"); - processor = factory.create(null, null, null, config); + processor = factory.create(null, null, null, config, null); assertThat(processor.getIndexNameFormatTemplate().newInstance(Map.of()).execute(), equalTo("yyyyMMdd")); config = new HashMap<>(); @@ -63,7 +63,7 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("date_rounding", "y"); config.put("timezone", "+02:00"); - processor = factory.create(null, null, null, config); + processor = factory.create(null, null, null, config, null); assertThat(processor.getTimezone(), equalTo(ZoneOffset.ofHours(2))); config = new HashMap<>(); @@ -71,7 +71,7 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("index_name_prefix", "_prefix"); config.put("date_rounding", "y"); - processor = factory.create(null, null, null, config); + processor = factory.create(null, null, null, config, null); assertThat(processor.getIndexNamePrefixTemplate().newInstance(Map.of()).execute(), equalTo("_prefix")); } @@ -79,12 +79,15 @@ public void testRequiredFields() throws Exception { DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance()); Map config = new HashMap<>(); config.put("date_rounding", "y"); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(null, null, null, config, null) + ); assertThat(e.getMessage(), equalTo("[field] required property is missing")); config.clear(); config.put("field", "_field"); - e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config, null)); assertThat(e.getMessage(), equalTo("[date_rounding] required property is missing")); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java index 76a219ec4458d..bb1326da0fe9a 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java @@ -38,7 +38,7 @@ public void testBuildDefaults() throws Exception { config.put("field", sourceField); config.put("formats", List.of("dd/MM/yyyyy")); String processorTag = randomAlphaOfLength(10); - DateProcessor processor = factory.create(null, processorTag, null, config); + DateProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(sourceField)); assertThat(processor.getTargetField(), equalTo(DateProcessor.DEFAULT_TARGET_FIELD)); @@ -54,7 +54,7 @@ public void testMatchFieldIsMandatory() throws Exception { config.put("formats", List.of("dd/MM/yyyyy")); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("processor creation should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), containsString("[field] required property is missing")); @@ -69,7 +69,7 @@ public void testMatchFormatsIsMandatory() throws Exception { config.put("target_field", targetField); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("processor creation should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), containsString("[formats] required property is missing")); @@ -84,7 +84,7 @@ public void testParseLocale() throws Exception { Locale locale = randomFrom(Locale.GERMANY, Locale.FRENCH, Locale.ROOT); config.put("locale", locale.toLanguageTag()); - DateProcessor processor = factory.create(null, null, null, config); + DateProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getLocale().newInstance(Map.of()).execute(), equalTo(locale.toLanguageTag())); } @@ -96,7 +96,7 @@ public void testParseTimezone() throws Exception { ZoneId timezone = randomZone(); config.put("timezone", timezone.getId()); - DateProcessor processor = factory.create(null, null, null, config); + DateProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getTimezone().newInstance(Map.of()).execute(), equalTo(timezone.getId())); } @@ -106,7 +106,7 @@ public void testParseMatchFormats() throws Exception { config.put("field", sourceField); config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy")); - DateProcessor processor = factory.create(null, null, null, config); + DateProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getFormats(), equalTo(List.of("dd/MM/yyyy", "dd-MM-yyyy"))); } @@ -117,7 +117,7 @@ public void testParseMatchFormatsFailure() throws Exception { config.put("formats", "dd/MM/yyyy"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("processor creation should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), containsString("[formats] property isn't a list, but of type [java.lang.String]")); @@ -132,7 +132,7 @@ public void testParseTargetField() throws Exception { config.put("target_field", targetField); config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy")); - DateProcessor processor = factory.create(null, null, null, config); + DateProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getTargetField(), equalTo(targetField)); } @@ -145,7 +145,7 @@ public void testParseOutputFormat() throws Exception { config.put("target_field", targetField); config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy")); config.put("output_format", outputFormat); - DateProcessor processor = factory.create(null, null, null, config); + DateProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getOutputFormat(), equalTo(outputFormat)); } @@ -156,7 +156,7 @@ public void testDefaultOutputFormat() throws Exception { config.put("field", sourceField); config.put("target_field", targetField); config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy")); - DateProcessor processor = factory.create(null, null, null, config); + DateProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getOutputFormat(), equalTo(DateProcessor.DEFAULT_OUTPUT_FORMAT)); } @@ -170,7 +170,7 @@ public void testInvalidOutputFormatRejected() throws Exception { config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy")); config.put("output_format", outputFormat); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, null, null, config)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, null, null, config, null)); assertThat(e.getMessage(), containsString("invalid output format [" + outputFormat + "]")); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java index 8f5278ea17a5a..a3eda38576c4b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java @@ -36,7 +36,7 @@ public void testCreate() { config.put("append_separator", appendSeparator); config.put("ignore_missing", true); - DissectProcessor processor = factory.create(null, processorTag, null, config); + DissectProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.field, equalTo(fieldName)); assertThat(processor.pattern, equalTo(pattern)); @@ -49,7 +49,7 @@ public void testCreateMissingField() { DissectProcessor.Factory factory = new DissectProcessor.Factory(); Map config = new HashMap<>(); config.put("pattern", "%{a},%{b},%{c}"); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null)); assertThat(e.getMessage(), equalTo("[field] required property is missing")); } @@ -57,7 +57,7 @@ public void testCreateMissingPattern() { DissectProcessor.Factory factory = new DissectProcessor.Factory(); Map config = new HashMap<>(); config.put("field", randomAlphaOfLength(10)); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null)); assertThat(e.getMessage(), equalTo("[pattern] required property is missing")); } @@ -66,7 +66,7 @@ public void testCreateMissingOptionals() { Map config = new HashMap<>(); config.put("pattern", "%{a},%{b},%{c}"); config.put("field", randomAlphaOfLength(10)); - DissectProcessor processor = factory.create(null, "_tag", null, config); + DissectProcessor processor = factory.create(null, "_tag", null, config, null); assertThat(processor.appendSeparator, equalTo("")); assertThat(processor.ignoreMissing, is(false)); } @@ -76,6 +76,6 @@ public void testCreateBadPattern() { Map config = new HashMap<>(); config.put("pattern", "no keys defined"); config.put("field", randomAlphaOfLength(10)); - expectThrows(DissectException.class, () -> factory.create(null, "_tag", null, config)); + expectThrows(DissectException.class, () -> factory.create(null, "_tag", null, config, null)); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java index 9b86111c3c40f..646cd2e0ce30b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java @@ -26,13 +26,13 @@ public void testCreate() throws Exception { Map config = new HashMap<>(); config.put("field", "_field.field"); config.put("path", "_path"); - DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config); + DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config, null); assertThat(processor.getField(), equalTo("_field.field")); assertThat(processor.getPath(), equalTo("_path")); config = new HashMap<>(); config.put("field", "_field.field"); - processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config); + processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config, null); assertThat(processor.getField(), equalTo("_field.field")); assertThat(processor.getPath(), nullValue()); } @@ -45,7 +45,7 @@ public void testValidFields() throws Exception { Map config = new HashMap<>(); config.put("field", field); config.put("path", "_path"); - DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config); + DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config, null); assertThat(processor.getField(), equalTo(field)); assertThat(processor.getPath(), equalTo("_path")); } @@ -56,7 +56,7 @@ public void testCreate_fieldMissing() throws Exception { Map config = new HashMap<>(); config.put("path", "_path"); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null)); assertThat(e.getMessage(), equalTo("[field] required property is missing")); } @@ -66,7 +66,7 @@ public void testCreate_invalidFields() throws Exception { for (String field : fields) { Map config = new HashMap<>(); config.put("field", field); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null)); assertThat(e.getMessage(), equalTo("[field] field does not contain a dot and is not a wildcard")); } @@ -74,7 +74,7 @@ public void testCreate_invalidFields() throws Exception { for (String field : fields) { Map config = new HashMap<>(); config.put("field", field); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null)); assertThat(e.getMessage(), equalTo("[field] Field can't start or end with a dot")); } @@ -82,7 +82,7 @@ public void testCreate_invalidFields() throws Exception { for (String field : fields) { Map config = new HashMap<>(); config.put("field", field); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null)); assertThat(e.getMessage(), equalTo("[field] No space between dots")); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java index 7df3cfb748dea..3dfbc1eceaccc 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java @@ -33,7 +33,7 @@ public void testCreate() throws Exception { Map config = new HashMap<>(); config.put("message", "error"); String processorTag = randomAlphaOfLength(10); - FailProcessor failProcessor = factory.create(null, processorTag, null, config); + FailProcessor failProcessor = factory.create(null, processorTag, null, config, null); assertThat(failProcessor.getTag(), equalTo(processorTag)); assertThat(failProcessor.getMessage().newInstance(Map.of()).execute(), equalTo("error")); } @@ -41,7 +41,7 @@ public void testCreate() throws Exception { public void testCreateMissingMessageField() throws Exception { Map config = new HashMap<>(); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[message] required property is missing")); @@ -55,7 +55,7 @@ public void testInvalidMustacheTemplate() throws Exception { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows( ElasticsearchException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java index 65d6eb96629dc..72a9032cd3c74 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java @@ -48,7 +48,7 @@ public void testCreate() throws Exception { config.put("ignore_missing", ignoreMissing); String processorTag = randomAlphaOfLength(10); - FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config); + FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config, null); assertThat(fingerprintProcessor.getTag(), equalTo(processorTag)); assertThat(fingerprintProcessor.getFields(), equalTo(sortedFieldList)); assertThat(fingerprintProcessor.getTargetField(), equalTo(targetField)); @@ -68,7 +68,7 @@ public void testMethod() throws Exception { config.put("method", method); String processorTag = randomAlphaOfLength(10); - FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config); + FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config, null); assertThat(fingerprintProcessor.getTag(), equalTo(processorTag)); assertThat(fingerprintProcessor.getFields(), equalTo(sortedFieldList)); assertThat(fingerprintProcessor.getThreadLocalHasher().get().getAlgorithm(), equalTo(method)); @@ -80,7 +80,7 @@ public void testMethod() throws Exception { ); config.put("fields", fieldList); config.put("method", invalidMethod); - ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); + ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null)); assertThat(e.getMessage(), containsString("[" + invalidMethod + "] must be one of the supported hash methods [")); } @@ -93,17 +93,17 @@ public void testFields() throws Exception { config.put("fields", fieldList); String processorTag = randomAlphaOfLength(10); - FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config); + FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config, null); assertThat(fingerprintProcessor.getTag(), equalTo(processorTag)); assertThat(fingerprintProcessor.getFields(), equalTo(sortedFieldList)); // fields is a list of length zero config.put("fields", List.of()); - ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); + ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null)); assertThat(e.getMessage(), containsString("must specify at least one field")); // fields is missing - e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config)); + e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null)); assertThat(e.getMessage(), containsString("[fields] required property is missing")); } @@ -115,7 +115,7 @@ public void testDefaults() throws Exception { HashMap config = new HashMap<>(); config.put("fields", fieldList); - FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config); + FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config, null); assertThat(fingerprintProcessor.getTag(), equalTo(processorTag)); assertThat(fingerprintProcessor.getFields(), equalTo(sortedFieldList)); assertThat(fingerprintProcessor.getTargetField(), equalTo(FingerprintProcessor.Factory.DEFAULT_TARGET)); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java index a687bdd5cf41b..42a61049c906b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java @@ -215,7 +215,7 @@ private String doTestFingerprint( if (salt != null) { config.put("salt", salt); } - FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config); + FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config, null); byte[] expectedBytes = new byte[0]; if (salt != null) { @@ -257,7 +257,7 @@ public void testMethod() throws Exception { config.put("fields", List.of("foo", "bar")); config.put("method", FingerprintProcessor.Factory.SUPPORTED_DIGESTS[k]); - FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config); + FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config, null); var input = TestIngestDocument.withDefaultVersion(inputMap); var output = fp.execute(input); assertTrue(output.hasField("fingerprint")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java index f5a38dab748f4..dee040b9a4fab 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java @@ -32,13 +32,13 @@ public class ForEachProcessorFactoryTests extends ESTestCase { public void testCreate() throws Exception { Processor processor = new TestProcessor(ingestDocument -> {}); Map registry = new HashMap<>(); - registry.put("_name", (r, t, description, c) -> processor); + registry.put("_name", (r, t, description, c, projectId) -> processor); ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService); Map config = new HashMap<>(); config.put("field", "_field"); config.put("processor", Map.of("_name", Collections.emptyMap())); - ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config); + ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config, null); assertThat(forEachProcessor, notNullValue()); assertThat(forEachProcessor.getField(), equalTo("_field")); assertThat(forEachProcessor.getInnerProcessor(), sameInstance(processor)); @@ -48,14 +48,14 @@ public void testCreate() throws Exception { public void testSetIgnoreMissing() throws Exception { Processor processor = new TestProcessor(ingestDocument -> {}); Map registry = new HashMap<>(); - registry.put("_name", (r, t, description, c) -> processor); + registry.put("_name", (r, t, description, c, projectId) -> processor); ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService); Map config = new HashMap<>(); config.put("field", "_field"); config.put("processor", Map.of("_name", Collections.emptyMap())); config.put("ignore_missing", true); - ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config); + ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config, null); assertThat(forEachProcessor, notNullValue()); assertThat(forEachProcessor.getField(), equalTo("_field")); assertThat(forEachProcessor.getInnerProcessor(), sameInstance(processor)); @@ -65,8 +65,8 @@ public void testSetIgnoreMissing() throws Exception { public void testCreateWithTooManyProcessorTypes() throws Exception { Processor processor = new TestProcessor(ingestDocument -> {}); Map registry = new HashMap<>(); - registry.put("_first", (r, t, description, c) -> processor); - registry.put("_second", (r, t, description, c) -> processor); + registry.put("_first", (r, t, description, c, projectId) -> processor); + registry.put("_second", (r, t, description, c, projectId) -> processor); ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService); Map config = new HashMap<>(); @@ -75,7 +75,10 @@ public void testCreateWithTooManyProcessorTypes() throws Exception { processorTypes.put("_first", Map.of()); processorTypes.put("_second", Map.of()); config.put("processor", processorTypes); - Exception exception = expectThrows(ElasticsearchParseException.class, () -> forEachFactory.create(registry, null, null, config)); + Exception exception = expectThrows( + ElasticsearchParseException.class, + () -> forEachFactory.create(registry, null, null, config, null) + ); assertThat(exception.getMessage(), equalTo("[processor] Must specify exactly one processor type")); } @@ -86,7 +89,7 @@ public void testCreateWithNonExistingProcessorType() throws Exception { config.put("processor", Map.of("_name", Collections.emptyMap())); Exception expectedException = expectThrows( ElasticsearchParseException.class, - () -> forEachFactory.create(Map.of(), null, null, config) + () -> forEachFactory.create(Map.of(), null, null, config, null) ); assertThat(expectedException.getMessage(), equalTo("No processor type exists with name [_name]")); } @@ -94,11 +97,11 @@ public void testCreateWithNonExistingProcessorType() throws Exception { public void testCreateWithMissingField() throws Exception { Processor processor = new TestProcessor(ingestDocument -> {}); Map registry = new HashMap<>(); - registry.put("_name", (r, t, description, c) -> processor); + registry.put("_name", (r, t, description, c, projectId) -> processor); ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService); Map config = new HashMap<>(); config.put("processor", List.of(Map.of("_name", Map.of()))); - Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(registry, null, null, config)); + Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(registry, null, null, config, null)); assertThat(exception.getMessage(), equalTo("[field] required property is missing")); } @@ -106,7 +109,7 @@ public void testCreateWithMissingProcessor() { ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService); Map config = new HashMap<>(); config.put("field", "_field"); - Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(Map.of(), null, null, config)); + Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(Map.of(), null, null, config, null)); assertThat(exception.getMessage(), equalTo("[processor] required property is missing")); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java index 2f5dc9618d0c3..d58ac252d95ab 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java @@ -30,7 +30,7 @@ public void testBuild() throws Exception { config.put("field", "_field"); config.put("patterns", List.of("(?\\w+)")); String processorTag = randomAlphaOfLength(10); - GrokProcessor processor = factory.create(null, processorTag, null, config); + GrokProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getMatchField(), equalTo("_field")); assertThat(processor.getGrok(), notNullValue()); @@ -45,7 +45,7 @@ public void testBuildWithIgnoreMissing() throws Exception { config.put("patterns", List.of("(?\\w+)")); config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - GrokProcessor processor = factory.create(null, processorTag, null, config); + GrokProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getMatchField(), equalTo("_field")); assertThat(processor.getGrok(), notNullValue()); @@ -56,7 +56,10 @@ public void testBuildMissingField() throws Exception { GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop()); Map config = new HashMap<>(); config.put("patterns", List.of("(?\\w+)")); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(null, null, null, config, null) + ); assertThat(e.getMessage(), equalTo("[field] required property is missing")); } @@ -64,7 +67,10 @@ public void testBuildMissingPatterns() throws Exception { GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop()); Map config = new HashMap<>(); config.put("field", "foo"); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(null, null, null, config, null) + ); assertThat(e.getMessage(), equalTo("[patterns] required property is missing")); } @@ -73,7 +79,10 @@ public void testBuildEmptyPatternsList() throws Exception { Map config = new HashMap<>(); config.put("field", "foo"); config.put("patterns", List.of()); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(null, null, null, config, null) + ); assertThat(e.getMessage(), equalTo("[patterns] List of patterns must not be empty")); } @@ -84,7 +93,7 @@ public void testCreateWithCustomPatterns() throws Exception { config.put("field", "_field"); config.put("patterns", List.of("%{MY_PATTERN:name}!")); config.put("pattern_definitions", Map.of("MY_PATTERN", "foo")); - GrokProcessor processor = factory.create(null, null, null, config); + GrokProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getMatchField(), equalTo("_field")); assertThat(processor.getGrok(), notNullValue()); assertThat(processor.getGrok().match("foo!"), equalTo(true)); @@ -95,7 +104,10 @@ public void testCreateWithInvalidPattern() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("patterns", List.of("[")); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(null, null, null, config, null) + ); assertThat(e.getMessage(), equalTo("[patterns] Invalid regex pattern found in: [[]. premature end of char-class")); } @@ -105,7 +117,10 @@ public void testCreateWithInvalidPatternDefinition() throws Exception { config.put("field", "_field"); config.put("patterns", List.of("%{MY_PATTERN:name}!")); config.put("pattern_definitions", Map.of("MY_PATTERN", "[")); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(null, null, null, config, null) + ); assertThat( e.getMessage(), equalTo("[patterns] Invalid regex pattern found in: [%{MY_PATTERN:name}!]. premature end of char-class") @@ -119,7 +134,10 @@ public void testCreateWithInvalidEcsCompatibilityMode() throws Exception { config.put("patterns", List.of("(?\\w+)")); String invalidEcsMode = randomAlphaOfLength(3); config.put("ecs_compatibility", invalidEcsMode); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(null, null, null, config, null) + ); assertThat(e.getMessage(), equalTo("[ecs_compatibility] unsupported mode '" + invalidEcsMode + "'")); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java index 489e3860f510a..2555575160ab7 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java @@ -44,7 +44,7 @@ public void testCreateNoPatternPresent() throws Exception { config.put("field", "field1"); config.put("replacement", "-"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[pattern] required property is missing")); @@ -57,7 +57,7 @@ public void testCreateNoReplacementPresent() throws Exception { config.put("field", "field1"); config.put("pattern", "\\."); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[replacement] required property is missing")); @@ -71,7 +71,7 @@ public void testCreateInvalidPattern() throws Exception { config.put("pattern", "["); config.put("replacement", "-"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), containsString("[pattern] Invalid regex pattern. Unclosed character class")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java index e1e5485173db2..05e50eee03d68 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java @@ -25,7 +25,7 @@ public void testCreate() throws Exception { config.put("field", "field1"); config.put("separator", "-"); String processorTag = randomAlphaOfLength(10); - JoinProcessor joinProcessor = factory.create(null, processorTag, null, config); + JoinProcessor joinProcessor = factory.create(null, processorTag, null, config, null); assertThat(joinProcessor.getTag(), equalTo(processorTag)); assertThat(joinProcessor.getField(), equalTo("field1")); assertThat(joinProcessor.getSeparator(), equalTo("-")); @@ -37,7 +37,7 @@ public void testCreateNoFieldPresent() throws Exception { Map config = new HashMap<>(); config.put("separator", "-"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -49,7 +49,7 @@ public void testCreateNoSeparatorPresent() throws Exception { Map config = new HashMap<>(); config.put("field", "field1"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[separator] required property is missing")); @@ -63,7 +63,7 @@ public void testCreateWithTargetField() throws Exception { config.put("separator", "-"); config.put("target_field", "target"); String processorTag = randomAlphaOfLength(10); - JoinProcessor joinProcessor = factory.create(null, processorTag, null, config); + JoinProcessor joinProcessor = factory.create(null, processorTag, null, config, null); assertThat(joinProcessor.getTag(), equalTo(processorTag)); assertThat(joinProcessor.getField(), equalTo("field1")); assertThat(joinProcessor.getSeparator(), equalTo("-")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java index 6e9346f19f732..e525b8a713259 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java @@ -31,7 +31,7 @@ public void testCreate() throws Exception { Map config = new HashMap<>(); config.put("field", randomField); config.put("target_field", randomTargetField); - JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config); + JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config, null); assertThat(jsonProcessor.getTag(), equalTo(processorTag)); assertThat(jsonProcessor.getField(), equalTo(randomField)); assertThat(jsonProcessor.getTargetField(), equalTo(randomTargetField)); @@ -43,7 +43,7 @@ public void testCreateWithAddToRoot() throws Exception { Map config = new HashMap<>(); config.put("field", randomField); config.put("add_to_root", true); - JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config); + JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config, null); assertThat(jsonProcessor.getTag(), equalTo(processorTag)); assertThat(jsonProcessor.getField(), equalTo(randomField)); assertThat(jsonProcessor.getTargetField(), equalTo(randomField)); @@ -55,7 +55,7 @@ public void testCreateWithDefaultTarget() throws Exception { String randomField = randomAlphaOfLength(10); Map config = new HashMap<>(); config.put("field", randomField); - JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config); + JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config, null); assertThat(jsonProcessor.getTag(), equalTo(processorTag)); assertThat(jsonProcessor.getField(), equalTo(randomField)); assertThat(jsonProcessor.getTargetField(), equalTo(randomField)); @@ -66,7 +66,7 @@ public void testCreateWithMissingField() throws Exception { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows( ElasticsearchParseException.class, - () -> FACTORY.create(null, processorTag, null, config) + () -> FACTORY.create(null, processorTag, null, config, null) ); assertThat(exception.getMessage(), equalTo("[field] required property is missing")); } @@ -79,7 +79,7 @@ public void testCreateWithStrictParsingParameter() throws Exception { { Map strictConfig = new HashMap<>(); strictConfig.put("field", fieldName); - JsonProcessor strictProcessor = FACTORY.create(null, processorTag, null, strictConfig); + JsonProcessor strictProcessor = FACTORY.create(null, processorTag, null, strictConfig, null); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> strictProcessor.execute(document)); assertThat(exception.getMessage(), containsString("is not valid JSON and the strict_json_parsing parameter is true")); } @@ -88,7 +88,7 @@ public void testCreateWithStrictParsingParameter() throws Exception { Map lenientConfig = new HashMap<>(); lenientConfig.put("field", fieldName); lenientConfig.put("strict_json_parsing", false); - JsonProcessor lenientProcessor = FACTORY.create(null, processorTag, null, lenientConfig); + JsonProcessor lenientProcessor = FACTORY.create(null, processorTag, null, lenientConfig, null); IngestDocument result = lenientProcessor.execute(document); assertThat(result.getSource().get(fieldName), equalTo(123)); } @@ -103,7 +103,7 @@ public void testCreateWithBothTargetFieldAndAddToRoot() throws Exception { config.put("add_to_root", true); ElasticsearchException exception = expectThrows( ElasticsearchParseException.class, - () -> FACTORY.create(null, randomAlphaOfLength(10), null, config) + () -> FACTORY.create(null, randomAlphaOfLength(10), null, config, null) ); assertThat(exception.getMessage(), equalTo("[target_field] Cannot set a target field while also setting `add_to_root` to true")); } @@ -151,6 +151,6 @@ private JsonProcessor getJsonProcessorWithMergeStrategy(String mergeStrategy, bo if (mergeStrategy != null) { config.put("add_to_root_conflict_strategy", mergeStrategy); } - return FACTORY.create(null, randomAlphaOfLength(10), null, config); + return FACTORY.create(null, randomAlphaOfLength(10), null, config, null); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java index f4f39aa66d381..c7efba7f5c893 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java @@ -40,7 +40,7 @@ public void testCreateWithDefaults() throws Exception { config.put("field_split", "&"); config.put("value_split", "="); String processorTag = randomAlphaOfLength(10); - KeyValueProcessor processor = factory.create(null, processorTag, null, config); + KeyValueProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField().newInstance(Map.of()).execute(), equalTo("field1")); assertThat(processor.getFieldSplit(), equalTo("&")); @@ -60,7 +60,7 @@ public void testCreateWithAllFieldsSet() throws Exception { config.put("exclude_keys", List.of()); config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - KeyValueProcessor processor = factory.create(null, processorTag, null, config); + KeyValueProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField().newInstance(Map.of()).execute(), equalTo("field1")); assertThat(processor.getFieldSplit(), equalTo("&")); @@ -76,7 +76,7 @@ public void testCreateWithMissingField() { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows( ElasticsearchParseException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(exception.getMessage(), equalTo("[field] required property is missing")); } @@ -87,7 +87,7 @@ public void testCreateWithMissingFieldSplit() { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows( ElasticsearchParseException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(exception.getMessage(), equalTo("[field_split] required property is missing")); } @@ -99,7 +99,7 @@ public void testCreateWithMissingValueSplit() { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows( ElasticsearchParseException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(exception.getMessage(), equalTo("[value_split] required property is missing")); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java index 9ae4be1efce7e..5bdfd08d3d04f 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java @@ -251,6 +251,6 @@ private static KeyValueProcessor createKvProcessor( if (prefix != null) { config.put("prefix", prefix); } - return FACTORY.create(null, randomAlphaOfLength(10), null, config); + return FACTORY.create(null, randomAlphaOfLength(10), null, config, null); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java index ed669a6687fa6..e5b81907818f3 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java @@ -50,7 +50,7 @@ public void testCreate() throws Exception { config.put("ignore_missing", ignoreMissing); String processorTag = randomAlphaOfLength(10); - NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config); + NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config, null); assertThat(networkProcessor.getTag(), equalTo(processorTag)); assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField)); assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField)); @@ -75,7 +75,7 @@ public void testCreateInternalNetworksField() throws Exception { config.put("ignore_missing", ignoreMissing); String processorTag = randomAlphaOfLength(10); - NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config); + NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config, null); assertThat(networkProcessor.getTag(), equalTo(processorTag)); assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField)); assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField)); @@ -88,7 +88,7 @@ public void testRequiredFields() throws Exception { HashMap config = new HashMap<>(); String processorTag = randomAlphaOfLength(10); try { - factory.create(null, processorTag, null, config); + factory.create(null, processorTag, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[internal_networks] or [internal_networks_field] must be specified")); @@ -102,7 +102,7 @@ public void testDefaultFields() throws Exception { internalNetworks.add("10.0.0.0/8"); config.put("internal_networks", internalNetworks); - NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config); + NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config, null); assertThat(networkProcessor.getTag(), equalTo(processorTag)); assertThat(networkProcessor.getSourceIpField(), equalTo(DEFAULT_SOURCE_IP)); assertThat(networkProcessor.getDestinationIpField(), equalTo(DEFAULT_DEST_IP)); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java index 2f4e159dfcd0a..af96c287b49a2 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java @@ -139,7 +139,8 @@ public void testReadFromField() throws Exception { null, processorTag, null, - config + config, + null ); IngestDocument input = TestIngestDocument.withDefaultVersion(source); IngestDocument output = processor.execute(input); @@ -158,7 +159,7 @@ public void testInternalNetworksAndField() throws Exception { config.put("internal_networks", networks); ElasticsearchParseException e = expectThrows( ElasticsearchParseException.class, - () -> new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config) + () -> new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config, null) ); assertThat( e.getMessage(), @@ -184,7 +185,8 @@ private void testNetworkDirectionProcessor( null, processorTag, null, - config + config, + null ); IngestDocument input = TestIngestDocument.withDefaultVersion(source); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java index 9928ed18bfbff..1dc7e87004caf 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java @@ -38,7 +38,7 @@ public void testCreate() throws Exception { config.put("ignore_missing", ignoreMissing); String processorTag = randomAlphaOfLength(10); - RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config); + RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config, null); assertThat(publicSuffixProcessor.getTag(), equalTo(processorTag)); assertThat(publicSuffixProcessor.getTargetField(), equalTo(targetField)); assertThat(publicSuffixProcessor.getIgnoreMissing(), equalTo(ignoreMissing)); @@ -51,7 +51,7 @@ public void testCreateDefaults() throws Exception { config.put("field", field); String processorTag = randomAlphaOfLength(10); - RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config); + RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config, null); assertThat(publicSuffixProcessor.getTargetField(), equalTo(RegisteredDomainProcessor.Factory.DEFAULT_TARGET_FIELD)); } @@ -59,7 +59,7 @@ public void testFieldRequired() throws Exception { HashMap config = new HashMap<>(); String processorTag = randomAlphaOfLength(10); try { - factory.create(null, processorTag, null, config); + factory.create(null, processorTag, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java index 0eeb2f6af53f5..f5c08854ca7d5 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java @@ -34,7 +34,7 @@ public void testCreate() throws Exception { Map config = new HashMap<>(); config.put("field", "field1"); String processorTag = randomAlphaOfLength(10); - RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config); + RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config, null); assertThat(removeProcessor.getTag(), equalTo(processorTag)); assertThat(removeProcessor.getFieldsToRemove().get(0).newInstance(Map.of()).execute(), equalTo("field1")); } @@ -43,7 +43,7 @@ public void testCreateKeepField() throws Exception { Map config = new HashMap<>(); config.put("keep", List.of("field1", "field2")); String processorTag = randomAlphaOfLength(10); - RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config); + RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config, null); assertThat(removeProcessor.getTag(), equalTo(processorTag)); assertThat(removeProcessor.getFieldsToKeep().get(0).newInstance(Map.of()).execute(), equalTo("field1")); assertThat(removeProcessor.getFieldsToKeep().get(1).newInstance(Map.of()).execute(), equalTo("field2")); @@ -53,7 +53,7 @@ public void testCreateMultipleFields() throws Exception { Map config = new HashMap<>(); config.put("field", List.of("field1", "field2")); String processorTag = randomAlphaOfLength(10); - RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config); + RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config, null); assertThat(removeProcessor.getTag(), equalTo(processorTag)); assertThat( removeProcessor.getFieldsToRemove().stream().map(template -> template.newInstance(Map.of()).execute()).toList(), @@ -64,7 +64,7 @@ public void testCreateMultipleFields() throws Exception { public void testCreateMissingField() throws Exception { Map config = new HashMap<>(); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[keep] or [field] must be specified")); @@ -76,7 +76,7 @@ public void testCreateTooManyFields() throws Exception { config.put("field", "field1"); config.put("keep", "field2"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[keep] and [field] cannot both be used in the same processor")); @@ -90,7 +90,7 @@ public void testInvalidMustacheTemplate() throws Exception { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows( ElasticsearchException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java index b18a7ff26e30e..2432235035ea4 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java @@ -46,7 +46,7 @@ public void testRemoveNonExistingField() throws Exception { Map config = new HashMap<>(); config.put("field", fieldName); String processorTag = randomAlphaOfLength(10); - Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config); + Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config, null); try { processor.execute(ingestDocument); fail("remove field should have failed"); @@ -62,7 +62,7 @@ public void testIgnoreMissing() throws Exception { config.put("field", fieldName); config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config); + Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config, null); processor.execute(ingestDocument); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java index d18ccace91424..ca10fd34a803b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java @@ -33,7 +33,7 @@ public void testCreate() throws Exception { config.put("field", "old_field"); config.put("target_field", "new_field"); String processorTag = randomAlphaOfLength(10); - RenameProcessor renameProcessor = factory.create(null, processorTag, null, config); + RenameProcessor renameProcessor = factory.create(null, processorTag, null, config, null); assertThat(renameProcessor.getTag(), equalTo(processorTag)); assertThat(renameProcessor.getField().newInstance(Map.of()).execute(), equalTo("old_field")); assertThat(renameProcessor.getTargetField().newInstance(Map.of()).execute(), equalTo("new_field")); @@ -47,7 +47,7 @@ public void testCreateWithIgnoreMissing() throws Exception { config.put("target_field", "new_field"); config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - RenameProcessor renameProcessor = factory.create(null, processorTag, null, config); + RenameProcessor renameProcessor = factory.create(null, processorTag, null, config, null); assertThat(renameProcessor.getTag(), equalTo(processorTag)); assertThat(renameProcessor.getField().newInstance(Map.of()).execute(), equalTo("old_field")); assertThat(renameProcessor.getTargetField().newInstance(Map.of()).execute(), equalTo("new_field")); @@ -60,7 +60,7 @@ public void testCreateWithEnableOverride() throws Exception { config.put("target_field", "new_field"); config.put("override", true); String processorTag = randomAlphaOfLength(10); - RenameProcessor renameProcessor = factory.create(null, processorTag, null, config); + RenameProcessor renameProcessor = factory.create(null, processorTag, null, config, null); assertThat(renameProcessor.getTag(), equalTo(processorTag)); assertThat(renameProcessor.getField().newInstance(Map.of()).execute(), equalTo("old_field")); assertThat(renameProcessor.getTargetField().newInstance(Map.of()).execute(), equalTo("new_field")); @@ -71,7 +71,7 @@ public void testCreateNoFieldPresent() throws Exception { Map config = new HashMap<>(); config.put("target_field", "new_field"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -82,7 +82,7 @@ public void testCreateNoToPresent() throws Exception { Map config = new HashMap<>(); config.put("field", "old_field"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[target_field] required property is missing")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorFactoryTests.java index e83093809cf01..4906cf7057cec 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorFactoryTests.java @@ -74,6 +74,6 @@ private static RerouteProcessor create(String dataset, String namespace) throws } private static RerouteProcessor create(Map config) throws Exception { - return new RerouteProcessor.Factory().create(null, null, null, new HashMap<>(config)); + return new RerouteProcessor.Factory().create(null, null, null, new HashMap<>(config), null); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java index 264ae5098203b..5e07909789d78 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java @@ -53,7 +53,7 @@ public void testFactoryValidationWithDefaultLang() throws Exception { Map configMap = new HashMap<>(); String randomType = randomFrom("id", "source"); configMap.put(randomType, "foo"); - ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap); + ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap, null); assertThat(processor.getScript().getLang(), equalTo(randomType.equals("id") ? null : Script.DEFAULT_SCRIPT_LANG)); assertThat(processor.getScript().getType().toString(), equalTo(INGEST_SCRIPT_PARAM_TO_TYPE.get(randomType))); assertThat(processor.getScript().getParams(), equalTo(Map.of())); @@ -69,7 +69,7 @@ public void testFactoryValidationWithParams() throws Exception { Map randomParams = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(10)); configMap.put(randomType, "foo"); configMap.put("params", randomParams); - ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap); + ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap, null); assertThat(processor.getScript().getLang(), equalTo(randomType.equals("id") ? null : Script.DEFAULT_SCRIPT_LANG)); assertThat(processor.getScript().getType().toString(), equalTo(INGEST_SCRIPT_PARAM_TO_TYPE.get(randomType))); assertThat(processor.getScript().getParams(), equalTo(randomParams)); @@ -83,7 +83,7 @@ public void testFactoryValidationForMultipleScriptingTypes() throws Exception { XContentParseException exception = expectThrows( XContentParseException.class, - () -> factory.create(null, randomAlphaOfLength(10), null, configMap) + () -> factory.create(null, randomAlphaOfLength(10), null, configMap, null) ); assertThat(exception.getMessage(), containsString("[script] failed to parse field [source]")); } @@ -94,7 +94,7 @@ public void testFactoryValidationAtLeastOneScriptingType() throws Exception { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> factory.create(null, randomAlphaOfLength(10), null, configMap) + () -> factory.create(null, randomAlphaOfLength(10), null, configMap, null) ); assertThat(exception.getMessage(), is("must specify either [source] for an inline script or [id] for a stored script")); @@ -108,7 +108,7 @@ public void testInlineBackcompat() throws Exception { Map configMap = new HashMap<>(); configMap.put("inline", "code"); - factory.create(null, randomAlphaOfLength(10), null, configMap); + factory.create(null, randomAlphaOfLength(10), null, configMap, null); assertWarnings("Deprecated field [inline] used, expected [source] instead"); } @@ -130,7 +130,7 @@ public void testFactoryInvalidateWithInvalidCompiledScript() throws Exception { ElasticsearchException exception = expectThrows( ElasticsearchException.class, - () -> factory.create(null, randomAlphaOfLength(10), null, configMap) + () -> factory.create(null, randomAlphaOfLength(10), null, configMap, null) ); assertThat(exception.getMessage(), is("compile-time exception")); @@ -151,7 +151,7 @@ public void testInlineIsCompiled() throws Exception { Map configMap = new HashMap<>(); configMap.put("source", scriptName); - ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap); + ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap, null); assertThat(processor.getScript().getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG)); assertThat(processor.getScript().getType(), equalTo(ScriptType.INLINE)); assertThat(processor.getScript().getParams(), equalTo(Map.of())); @@ -167,7 +167,7 @@ public void testStoredIsNotCompiled() throws Exception { factory = new ScriptProcessor.Factory(mockedScriptService); Map configMap = new HashMap<>(); configMap.put("id", "script_name"); - ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap); + ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap, null); assertNull(processor.getScript().getLang()); assertThat(processor.getScript().getType(), equalTo(ScriptType.STORED)); assertThat(processor.getScript().getParams(), equalTo(Map.of())); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java index 4d57967980cf1..cb1c30e295749 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java @@ -37,7 +37,7 @@ public void testCreate() throws Exception { config.put("field", "field1"); config.put("value", "value1"); String processorTag = randomAlphaOfLength(10); - SetProcessor setProcessor = factory.create(null, processorTag, null, config); + SetProcessor setProcessor = factory.create(null, processorTag, null, config, null); assertThat(setProcessor.getTag(), equalTo(processorTag)); assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1")); assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1")); @@ -52,7 +52,7 @@ public void testCreateWithOverride() throws Exception { config.put("value", "value1"); config.put("override", overrideEnabled); String processorTag = randomAlphaOfLength(10); - SetProcessor setProcessor = factory.create(null, processorTag, null, config); + SetProcessor setProcessor = factory.create(null, processorTag, null, config, null); assertThat(setProcessor.getTag(), equalTo(processorTag)); assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1")); assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1")); @@ -66,7 +66,7 @@ public void testCreateWithIgnoreEmptyValue() throws Exception { config.put("value", "value1"); config.put("ignore_empty_value", ignoreEmptyValueEnabled); String processorTag = randomAlphaOfLength(10); - SetProcessor setProcessor = factory.create(null, processorTag, null, config); + SetProcessor setProcessor = factory.create(null, processorTag, null, config, null); assertThat(setProcessor.getTag(), equalTo(processorTag)); assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1")); assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1")); @@ -77,7 +77,7 @@ public void testCreateNoFieldPresent() throws Exception { Map config = new HashMap<>(); config.put("value", "value1"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -88,7 +88,7 @@ public void testCreateNoValuePresent() throws Exception { Map config = new HashMap<>(); config.put("field", "field1"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[value] required property is missing")); @@ -100,7 +100,7 @@ public void testCreateNullValue() throws Exception { config.put("field", "field1"); config.put("value", null); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[value] required property is missing")); @@ -115,7 +115,7 @@ public void testInvalidMustacheTemplate() throws Exception { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows( ElasticsearchException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); @@ -126,7 +126,7 @@ public void testCreateWithCopyFrom() throws Exception { config.put("field", "field1"); config.put("copy_from", "field2"); String processorTag = randomAlphaOfLength(10); - SetProcessor setProcessor = factory.create(null, processorTag, null, config); + SetProcessor setProcessor = factory.create(null, processorTag, null, config, null); assertThat(setProcessor.getTag(), equalTo(processorTag)); assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1")); assertThat(setProcessor.getCopyFrom(), equalTo("field2")); @@ -140,7 +140,7 @@ public void testCreateWithCopyFromAndValue() throws Exception { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows( ElasticsearchException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(exception.getMessage(), equalTo("[copy_from] cannot set both `copy_from` and `value` in the same processor")); } @@ -153,7 +153,7 @@ public void testMediaType() throws Exception { config.put("value", "value1"); config.put("media_type", expectedMediaType); String processorTag = randomAlphaOfLength(10); - SetProcessor setProcessor = factory.create(null, processorTag, null, config); + SetProcessor setProcessor = factory.create(null, processorTag, null, config, null); assertThat(setProcessor.getTag(), equalTo(processorTag)); // invalid media type @@ -165,7 +165,10 @@ public void testMediaType() throws Exception { config2.put("field", "field1"); config2.put("value", "value1"); config2.put("media_type", expectedMediaType); - ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config2)); + ElasticsearchException e = expectThrows( + ElasticsearchException.class, + () -> factory.create(null, processorTag, null, config2, null) + ); assertThat(e.getMessage(), containsString("property does not contain a supported media type [" + expectedMediaType + "]")); } @@ -176,7 +179,7 @@ public void testCreateWithEmptyField() throws Exception { config.put("field", ""); config.put("value", "value1"); String processorTag = randomAlphaOfLength(10); - SetProcessor setProcessor = factory.create(null, processorTag, null, config); + SetProcessor setProcessor = factory.create(null, processorTag, null, config, null); assertThat(setProcessor.getTag(), equalTo(processorTag)); assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("")); assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java index eaa720003579b..489024a65811e 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java @@ -28,7 +28,7 @@ public void testCreate() throws Exception { config.put("field", fieldName); SortProcessor.Factory factory = new SortProcessor.Factory(); - SortProcessor processor = factory.create(null, processorTag, null, config); + SortProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(fieldName)); assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.ASCENDING)); @@ -44,7 +44,7 @@ public void testCreateWithOrder() throws Exception { config.put("order", "desc"); SortProcessor.Factory factory = new SortProcessor.Factory(); - SortProcessor processor = factory.create(null, processorTag, null, config); + SortProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(fieldName)); assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.DESCENDING)); @@ -61,7 +61,7 @@ public void testCreateWithTargetField() throws Exception { config.put("target_field", targetFieldName); SortProcessor.Factory factory = new SortProcessor.Factory(); - SortProcessor processor = factory.create(null, processorTag, null, config); + SortProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo(fieldName)); assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.ASCENDING)); @@ -78,7 +78,7 @@ public void testCreateWithInvalidOrder() throws Exception { SortProcessor.Factory factory = new SortProcessor.Factory(); try { - factory.create(null, processorTag, null, config); + factory.create(null, processorTag, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[order] Sort direction [invalid] not recognized. Valid values are: [asc, desc]")); @@ -89,7 +89,7 @@ public void testCreateMissingField() throws Exception { SortProcessor.Factory factory = new SortProcessor.Factory(); Map config = new HashMap<>(); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java index d44bad8b119d5..f61be8d0112dd 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java @@ -25,7 +25,7 @@ public void testCreate() throws Exception { config.put("field", "field1"); config.put("separator", "\\."); String processorTag = randomAlphaOfLength(10); - SplitProcessor splitProcessor = factory.create(null, processorTag, null, config); + SplitProcessor splitProcessor = factory.create(null, processorTag, null, config, null); assertThat(splitProcessor.getTag(), equalTo(processorTag)); assertThat(splitProcessor.getField(), equalTo("field1")); assertThat(splitProcessor.getSeparator(), equalTo("\\.")); @@ -38,7 +38,7 @@ public void testCreateNoFieldPresent() throws Exception { Map config = new HashMap<>(); config.put("separator", "\\."); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -50,7 +50,7 @@ public void testCreateNoSeparatorPresent() throws Exception { Map config = new HashMap<>(); config.put("field", "field1"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[separator] required property is missing")); @@ -64,7 +64,7 @@ public void testCreateWithTargetField() throws Exception { config.put("separator", "\\."); config.put("target_field", "target"); String processorTag = randomAlphaOfLength(10); - SplitProcessor splitProcessor = factory.create(null, processorTag, null, config); + SplitProcessor splitProcessor = factory.create(null, processorTag, null, config, null); assertThat(splitProcessor.getTag(), equalTo(processorTag)); assertThat(splitProcessor.getField(), equalTo("field1")); assertThat(splitProcessor.getSeparator(), equalTo("\\.")); @@ -81,7 +81,7 @@ public void testCreateWithPreserveTrailing() throws Exception { config.put("target_field", "target"); config.put("preserve_trailing", true); String processorTag = randomAlphaOfLength(10); - SplitProcessor splitProcessor = factory.create(null, processorTag, null, config); + SplitProcessor splitProcessor = factory.create(null, processorTag, null, config, null); assertThat(splitProcessor.getTag(), equalTo(processorTag)); assertThat(splitProcessor.getField(), equalTo("field1")); assertThat(splitProcessor.getSeparator(), equalTo("\\.")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java index 8fc8dad61ae2b..0583294b4c1c7 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java @@ -97,7 +97,7 @@ public void testSplitAppendable() throws Exception { Map splitConfig = new HashMap<>(); splitConfig.put("field", "flags"); splitConfig.put("separator", "\\|"); - Processor splitProcessor = (new SplitProcessor.Factory()).create(null, null, null, splitConfig); + Processor splitProcessor = (new SplitProcessor.Factory()).create(null, null, null, splitConfig, null); Map source = new HashMap<>(); source.put("flags", "new|hot|super|fun|interesting"); IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(source); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java index 971a726869fc0..36c86ea0a75ec 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java @@ -45,7 +45,7 @@ public void testCreate() throws Exception { config.put("ignore_missing", ignoreMissing); String processorTag = randomAlphaOfLength(10); - UriPartsProcessor uriPartsProcessor = factory.create(null, processorTag, null, config); + UriPartsProcessor uriPartsProcessor = factory.create(null, processorTag, null, config, null); assertThat(uriPartsProcessor.getTag(), equalTo(processorTag)); assertThat(uriPartsProcessor.getField(), equalTo(field)); assertThat(uriPartsProcessor.getTargetField(), equalTo(targetField)); @@ -58,7 +58,7 @@ public void testCreateNoFieldPresent() throws Exception { Map config = new HashMap<>(); config.put("value", "value1"); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); @@ -69,7 +69,7 @@ public void testCreateNullField() throws Exception { Map config = new HashMap<>(); config.put("field", null); try { - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[field] required property is missing")); diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index daba9c4e5e156..ea8d278c4d202 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -811,7 +811,7 @@ public static final class NonGeoProcessorsPlugin extends Plugin implements Inges @Override public Map getProcessors(Processor.Parameters parameters) { Map procMap = new HashMap<>(); - procMap.put(NON_GEO_PROCESSOR_TYPE, (factories, tag, description, config) -> { + procMap.put(NON_GEO_PROCESSOR_TYPE, (factories, tag, description, config, projectId) -> { readStringProperty(NON_GEO_PROCESSOR_TYPE, tag, config, "randomField"); return new AbstractProcessor(tag, description) { @Override diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java index d41f4ac341724..7c92636d3420a 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java @@ -74,12 +74,19 @@ public void test() throws Exception { databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb")); lazyLoadReaders(databaseNodeService); - final GeoIpProcessor processor1 = (GeoIpProcessor) factory.create(null, "_tag", null, new HashMap<>(Map.of("field", "_field"))); + final GeoIpProcessor processor1 = (GeoIpProcessor) factory.create( + null, + "_tag", + null, + new HashMap<>(Map.of("field", "_field")), + null + ); final GeoIpProcessor processor2 = (GeoIpProcessor) factory.create( null, "_tag", null, - new HashMap<>(Map.of("field", "_field", "database_file", "GeoLite2-City-Test.mmdb")) + new HashMap<>(Map.of("field", "_field", "database_file", "GeoLite2-City-Test.mmdb")), + null ); final AtomicBoolean completed = new AtomicBoolean(false); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 9e0392b2b7974..5b36e98b66f2f 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.geoip; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; @@ -227,7 +228,8 @@ public Processor create( final Map registry, final String processorTag, final String description, - final Map config + final Map config, + final ProjectId projectId ) throws IOException { String ipField = readStringProperty(type, processorTag, config, "field"); String targetField = readStringProperty(type, processorTag, config, "target_field", type); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index 7908c5a6741bc..65b4c52a29e0f 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -95,7 +95,7 @@ public void testBuildDefaults() throws Exception { config.put("field", "_field"); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("geoip")); @@ -112,7 +112,7 @@ public void testSetIgnoreMissing() throws Exception { config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("geoip")); @@ -129,7 +129,7 @@ public void testCountryBuildDefaults() throws Exception { config.put("database_file", "GeoLite2-Country.mmdb"); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); @@ -147,7 +147,7 @@ public void testAsnBuildDefaults() throws Exception { config.put("database_file", "GeoLite2-ASN.mmdb"); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); @@ -162,7 +162,7 @@ public void testBuildTargetField() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("target_field", "_field"); - GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("_field")); assertFalse(processor.isIgnoreMissing()); @@ -173,7 +173,7 @@ public void testBuildDbFile() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-Country.mmdb"); - GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("geoip")); assertThat(processor.getDatabaseType(), equalTo("GeoLite2-Country")); @@ -190,7 +190,7 @@ public void testBuildWithCountryDbAndAsnFields() { asnOnlyProperties.remove(Property.IP); String asnProperty = RandomPicks.randomFrom(Randomness.get(), asnOnlyProperties).toString(); config.put("properties", List.of(asnProperty)); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config, null)); assertThat( e.getMessage(), equalTo( @@ -211,7 +211,7 @@ public void testBuildWithAsnDbAndCityFields() { cityOnlyProperties.remove(Property.IP); String cityProperty = RandomPicks.randomFrom(Randomness.get(), cityOnlyProperties).toString(); config.put("properties", List.of(cityProperty)); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config, null)); assertThat( e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty + "]. valid values are [IP, ASN, ORGANIZATION_NAME, NETWORK]") @@ -226,7 +226,7 @@ public void testBuildNonExistingDbFile() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "does-not-exist.mmdb"); - Processor processor = factory.create(null, null, null, config); + Processor processor = factory.create(null, null, null, config, null); assertThat(processor, instanceOf(GeoIpProcessor.DatabaseUnavailableProcessor.class)); } @@ -237,7 +237,7 @@ public void testBuildBuiltinDatabaseMissing() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", randomFrom(DEFAULT_DATABASES)); - Processor processor = factory.create(null, null, null, config); + Processor processor = factory.create(null, null, null, config, null); assertThat(processor, instanceOf(GeoIpProcessor.DatabaseUnavailableProcessor.class)); } @@ -259,7 +259,7 @@ public void testBuildFields() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("properties", fieldNames); - GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(properties)); assertFalse(processor.isIgnoreMissing()); @@ -271,7 +271,7 @@ public void testBuildIllegalFieldOption() { Map config1 = new HashMap<>(); config1.put("field", "_field"); config1.put("properties", List.of("invalid")); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1, null)); assertThat( e.getMessage(), equalTo( @@ -285,7 +285,7 @@ public void testBuildIllegalFieldOption() { Map config2 = new HashMap<>(); config2.put("field", "_field"); config2.put("properties", "invalid"); - e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config2)); + e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config2, null)); assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]")); } @@ -301,7 +301,7 @@ public void testBuildUnsupportedDatabase() throws Exception { Map config1 = new HashMap<>(); config1.put("field", "_field"); config1.put("properties", List.of("ip")); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1, null)); assertThat( e.getMessage(), equalTo("[database_file] Unsupported database type [some-unsupported-database] for file [GeoLite2-City.mmdb]") @@ -320,7 +320,7 @@ public void testBuildNullDatabase() throws Exception { Map config1 = new HashMap<>(); config1.put("field", "_field"); config1.put("properties", List.of("ip")); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1, null)); assertThat(e.getMessage(), equalTo("[database_file] Unsupported database type [null] for file [GeoLite2-City.mmdb]")); } @@ -336,7 +336,7 @@ public void testStrictMaxmindSupport() throws Exception { config1.put("database_file", "some-ipinfo-database.mmdb"); config1.put("field", "_field"); config1.put("properties", List.of("ip")); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1)); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1, null)); assertThat( e.getMessage(), equalTo( @@ -358,7 +358,7 @@ public void testLaxMaxmindSupport() throws Exception { config1.put("database_file", "some-custom-database.mmdb"); config1.put("field", "_field"); config1.put("properties", List.of("ip")); - factory.create(null, null, null, config1); + factory.create(null, null, null, config1, null); assertWarnings(GeoIpProcessor.UNSUPPORTED_DATABASE_DEPRECATION_MESSAGE.replaceAll("\\{}", "some_custom_database.mmdb-City")); } @@ -393,7 +393,7 @@ public void testLazyLoading() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-City.mmdb"); - final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config); + final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config, null); // these are lazy loaded until first use, so we expect null here assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-City.mmdb").databaseReader.get()); @@ -404,7 +404,7 @@ public void testLazyLoading() throws Exception { config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-Country.mmdb"); - final GeoIpProcessor country = (GeoIpProcessor) factory.create(null, "_tag", null, config); + final GeoIpProcessor country = (GeoIpProcessor) factory.create(null, "_tag", null, config, null); // these are lazy loaded until first use, so we expect null here assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-Country.mmdb").databaseReader.get()); @@ -415,7 +415,7 @@ public void testLazyLoading() throws Exception { config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-ASN.mmdb"); - final GeoIpProcessor asn = (GeoIpProcessor) factory.create(null, "_tag", null, config); + final GeoIpProcessor asn = (GeoIpProcessor) factory.create(null, "_tag", null, config, null); // these are lazy loaded until first use, so we expect null here assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-ASN.mmdb").databaseReader.get()); @@ -462,7 +462,7 @@ public void testLoadingCustomDatabase() throws IOException { Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoIP2-City.mmdb"); - final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config); + final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config, null); // these are lazy loaded until first use, so we expect null here assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoIP2-City.mmdb").databaseReader.get()); @@ -478,7 +478,7 @@ public void testDownloadDatabaseOnPipelineCreation() throws IOException { Map config = new HashMap<>(); config.put("field", randomIdentifier()); config.put("download_database_on_pipeline_creation", randomBoolean()); - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); // Check all the config params were consumed. assertThat(config, anEmptyMap()); } @@ -498,7 +498,7 @@ public void testDefaultDatabaseWithTaskPresent() throws Exception { config.put("field", "_field"); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null); processor.execute(RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(Map.of("_field", "89.160.20.128")))); } @@ -507,7 +507,7 @@ public void testUpdateDatabaseWhileIngesting() throws Exception { GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "source_field"); - GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null); Map document = Map.of("source_field", "89.160.20.128"); { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(document)); @@ -558,7 +558,8 @@ public void testDatabaseNotReadyYet() throws Exception { null, null, null, - config + config, + null ); processor.execute(ingestDocument); assertThat(ingestDocument.getSourceAndMetadata().get("geoip"), nullValue()); @@ -580,7 +581,7 @@ public void testDatabaseNotReadyYet() throws Exception { document.put("source_field", "89.160.20.128"); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null); processor.execute(ingestDocument); assertThat(ingestDocument.getSourceAndMetadata().get("tags"), nullValue()); Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("geoip"); diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index ddd754c9f7d1c..0d4e8a06b62d4 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.useragent; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.UpdateForV10; @@ -191,7 +192,8 @@ public UserAgentProcessor create( Map factories, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) { String field = readStringProperty(TYPE, processorTag, config, "field"); String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "user_agent"); diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java index b08531cf80ec5..e74a2fdc97a4f 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java @@ -74,7 +74,7 @@ public void testBuildDefaults() throws Exception { String processorTag = randomAlphaOfLength(10); - UserAgentProcessor processor = factory.create(null, processorTag, null, config); + UserAgentProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("user_agent")); @@ -95,7 +95,7 @@ public void testBuildWithIgnoreMissing() throws Exception { String processorTag = randomAlphaOfLength(10); - UserAgentProcessor processor = factory.create(null, processorTag, null, config); + UserAgentProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("user_agent")); @@ -113,7 +113,7 @@ public void testBuildTargetField() throws Exception { config.put("field", "_field"); config.put("target_field", "_target_field"); - UserAgentProcessor processor = factory.create(null, null, null, config); + UserAgentProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("_target_field")); } @@ -125,7 +125,7 @@ public void testBuildRegexFile() throws Exception { config.put("field", "_field"); config.put("regex_file", regexWithoutDevicesFilename); - UserAgentProcessor processor = factory.create(null, null, null, config); + UserAgentProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0)); assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0)); @@ -140,7 +140,7 @@ public void testBuildExtractDeviceType() throws Exception { config.put("field", "_field"); config.put("extract_device_type", extractDeviceType); - UserAgentProcessor processor = factory.create(null, null, null, config); + UserAgentProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.isExtractDeviceType(), equalTo(extractDeviceType)); } @@ -152,7 +152,10 @@ public void testBuildNonExistingRegexFile() throws Exception { config.put("field", "_field"); config.put("regex_file", "does-not-exist.yml"); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(null, null, null, config, null) + ); assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yml] doesn't exist (has to exist at node startup)")); } @@ -172,7 +175,7 @@ public void testBuildFields() throws Exception { config.put("field", "_field"); config.put("properties", fieldNames); - UserAgentProcessor processor = factory.create(null, null, null, config); + UserAgentProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(properties)); } @@ -184,7 +187,10 @@ public void testInvalidProperty() throws Exception { config.put("field", "_field"); config.put("properties", Collections.singletonList("invalid")); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(null, null, null, config, null) + ); assertThat( e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " + "ORIGINAL, VERSION]") @@ -198,7 +204,10 @@ public void testInvalidPropertiesType() throws Exception { config.put("field", "_field"); config.put("properties", "invalid"); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(null, null, null, config, null) + ); assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]")); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java index 4d1ed9bce6440..c0b7e0ea5507e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java @@ -358,7 +358,7 @@ public Map getProcessors(Processor.Parameters paramet "final", getFinal(parameters, randomBoolean()), "request", - (processorFactories, tag, description, config) -> new AbstractProcessor(tag, description) { + (processorFactories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) { @Override public IngestDocument execute(final IngestDocument ingestDocument) throws Exception { ingestDocument.setFieldValue("request", true); @@ -371,7 +371,7 @@ public String getType() { } }, "changing_dest", - (processorFactories, tag, description, config) -> new AbstractProcessor(tag, description) { + (processorFactories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) { @Override public IngestDocument execute(final IngestDocument ingestDocument) throws Exception { ingestDocument.setFieldValue(IngestDocument.Metadata.INDEX.getFieldName(), "target"); @@ -385,7 +385,7 @@ public String getType() { }, "reroute", - (processorFactories, tag, description, config) -> { + (processorFactories, tag, description, config, projectId) -> { final String dest = Objects.requireNonNullElse( ConfigurationUtils.readOptionalStringProperty(description, tag, config, "dest"), "target" @@ -408,7 +408,7 @@ public String getType() { } private static Processor.Factory getDefault(Processor.Parameters parameters, boolean async) { - return (factories, tag, description, config) -> new AbstractProcessor(tag, description) { + return (factories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) { @Override public void execute(IngestDocument ingestDocument, BiConsumer handler) { @@ -446,7 +446,7 @@ public boolean isAsync() { } private static Processor.Factory getFinal(Processor.Parameters parameters, boolean async) { - return (processorFactories, tag, description, config) -> { + return (processorFactories, tag, description, config, projectId) -> { final String exists = (String) config.remove("exists"); return new AbstractProcessor(tag, description) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java index 7dc04811d2cd6..fa4c8abb0bd45 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java @@ -96,7 +96,7 @@ public Collection createComponents(PluginServices services) { @Override public Map getProcessors(Processor.Parameters parameters) { - return Map.of("test-async", (factories, tag, description, config) -> new AbstractProcessor(tag, description) { + return Map.of("test-async", (factories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) { @Override public void execute(IngestDocument ingestDocument, BiConsumer handler) { @@ -129,7 +129,7 @@ public boolean isAsync() { return true; } - }, "test", (processorFactories, tag, description, config) -> new AbstractProcessor(tag, description) { + }, "test", (processorFactories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) { @Override public IngestDocument execute(IngestDocument ingestDocument) throws Exception { String id = (String) ingestDocument.getSourceAndMetadata().get("_id"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java index 81a39dbe1f9f7..bcf7898a3dcb6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java @@ -386,11 +386,16 @@ public Map getProcessors(Processor.Parameters paramet factories.put(PipelineProcessor.TYPE, new PipelineProcessor.Factory(parameters.ingestService)); factories.put( "fail", - (processorFactories, tag, description, config) -> new TestProcessor(tag, "fail", description, new RuntimeException()) + (processorFactories, tag, description, config, projectId) -> new TestProcessor( + tag, + "fail", + description, + new RuntimeException() + ) ); factories.put( "onfailure_processor", - (processorFactories, tag, description, config) -> new TestProcessor(tag, "fail", description, document -> { + (processorFactories, tag, description, config, projectId) -> new TestProcessor(tag, "fail", description, document -> { String onFailurePipeline = document.getFieldValue("_ingest.on_failure_pipeline", String.class); document.setFieldValue("readme", "pipeline with id [" + onFailurePipeline + "] is a bad pipeline"); }) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java index 6f20b3c40cb34..aba9914097deb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java @@ -262,7 +262,7 @@ public static class CustomIngestTestPlugin extends IngestTestPlugin { @Override public Map getProcessors(Processor.Parameters parameters) { Map processors = new HashMap<>(); - processors.put("test", (factories, tag, description, config) -> { + processors.put("test", (factories, tag, description, config, projectId) -> { String field = (String) config.remove("field"); String value = (String) config.remove("value"); return new FakeProcessor("test", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java index 42e6d913d42b4..9284c8e2dfbc4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java @@ -166,7 +166,7 @@ public static class CustomIngestTestPlugin extends IngestTestPlugin { @Override public Map getProcessors(Processor.Parameters parameters) { Map processors = new HashMap<>(); - processors.put("set", (factories, tag, description, config) -> { + processors.put("set", (factories, tag, description, config, projectId) -> { String field = (String) config.remove("field"); String value = (String) config.remove("value"); return new FakeProcessor("set", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value)); diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index f7415b438dfae..258aa71d00d12 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -149,14 +149,20 @@ static Parsed parseWithPipelineId( return new Parsed(pipeline, ingestDocumentList, verbose); } - static Parsed parse(Map config, boolean verbose, IngestService ingestService, RestApiVersion restApiVersion) - throws Exception { + static Parsed parse( + ProjectId projectId, + Map config, + boolean verbose, + IngestService ingestService, + RestApiVersion restApiVersion + ) throws Exception { Map pipelineConfig = ConfigurationUtils.readMap(null, null, config, Fields.PIPELINE); Pipeline pipeline = Pipeline.create( SIMULATED_PIPELINE_ID, pipelineConfig, ingestService.getProcessorFactories(), - ingestService.getScriptService() + ingestService.getScriptService(), + projectId ); List ingestDocumentList = parseDocs(config, restApiVersion); return new Parsed(pipeline, ingestDocumentList, verbose); diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java index 6d2f2d8044388..f8e7668379afe 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java @@ -113,6 +113,7 @@ protected void doExecute(Task task, SimulatePipelineRequest request, ActionListe ); } else { simulateRequest = SimulatePipelineRequest.parse( + projectId, source, request.isVerbose(), ingestService, diff --git a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java index 97a68d9807688..ce59a6503735a 100644 --- a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java +++ b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java @@ -12,6 +12,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.Maps; @@ -487,7 +488,8 @@ public static ElasticsearchException newConfigurationException( public static List readProcessorConfigs( List> processorConfigs, ScriptService scriptService, - Map processorFactories + Map processorFactories, + ProjectId projectId ) throws Exception { Exception exception = null; List processors = new ArrayList<>(); @@ -498,7 +500,7 @@ public static List readProcessorConfigs( if (entry.getValue() == null) { throw newConfigurationException(entry.getKey(), null, null, "processor config cannot be [null]"); } else { - processors.add(readProcessor(processorFactories, scriptService, entry.getKey(), entry.getValue())); + processors.add(readProcessor(processorFactories, scriptService, entry.getKey(), entry.getValue(), projectId)); } } catch (Exception e) { exception = ExceptionsHelper.useOrSuppress(exception, e); @@ -575,14 +577,15 @@ public static Processor readProcessor( Map processorFactories, ScriptService scriptService, String type, - Object config + Object config, + ProjectId projectId ) throws Exception { if (config instanceof Map) { - return readProcessor(processorFactories, scriptService, type, (Map) config); + return readProcessor(processorFactories, scriptService, type, (Map) config, projectId); } else if (config instanceof String && "script".equals(type)) { Map normalizedScript = Maps.newMapWithExpectedSize(1); normalizedScript.put(ScriptType.INLINE.getParseField().getPreferredName(), config); - return readProcessor(processorFactories, scriptService, type, normalizedScript); + return readProcessor(processorFactories, scriptService, type, normalizedScript, projectId); } else { throw newConfigurationException(type, null, null, "property isn't a map, but of type [" + config.getClass().getName() + "]"); } @@ -592,7 +595,8 @@ public static Processor readProcessor( Map processorFactories, ScriptService scriptService, String type, - Map config + Map config, + ProjectId projectId ) throws Exception { String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY); String description = ConfigurationUtils.readOptionalStringProperty(null, tag, config, DESCRIPTION_KEY); @@ -607,14 +611,19 @@ public static Processor readProcessor( Pipeline.ON_FAILURE_KEY ); - List onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, scriptService, processorFactories); + List onFailureProcessors = readProcessorConfigs( + onFailureProcessorConfigs, + scriptService, + processorFactories, + projectId + ); if (onFailureProcessorConfigs != null && onFailureProcessors.isEmpty()) { throw newConfigurationException(type, tag, Pipeline.ON_FAILURE_KEY, "processors list cannot be empty"); } try { - Processor processor = factory.create(processorFactories, tag, description, config); + Processor processor = factory.create(processorFactories, tag, description, config, projectId); if (config.isEmpty() == false) { throw new ElasticsearchParseException( "processor [{}] doesn't support one or more provided configuration parameters {}", diff --git a/server/src/main/java/org/elasticsearch/ingest/DropProcessor.java b/server/src/main/java/org/elasticsearch/ingest/DropProcessor.java index 829fa4c814b25..9a1d7d5587234 100644 --- a/server/src/main/java/org/elasticsearch/ingest/DropProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/DropProcessor.java @@ -9,6 +9,8 @@ package org.elasticsearch.ingest; +import org.elasticsearch.cluster.metadata.ProjectId; + import java.util.Map; /** @@ -40,7 +42,8 @@ public Processor create( final Map processorFactories, final String tag, final String description, - final Map config + final Map config, + final ProjectId projectId ) { return new DropProcessor(tag, description); } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index d2dcc8762971a..b4d0df780a0ff 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -372,6 +372,10 @@ public ScriptService getScriptService() { return scriptService; } + public ProjectResolver getProjectResolver() { + return projectResolver; + } + /** * Deletes the pipeline specified by id in the request. */ @@ -522,7 +526,7 @@ public void putPipeline( } nodeInfoListener.accept(listener.delegateFailureAndWrap((l, nodeInfos) -> { - validatePipelineRequest(request, nodeInfos); + validatePipelineRequest(projectId, request, nodeInfos); taskQueue.submitTask( "put-pipeline-" + request.getId(), @@ -532,14 +536,14 @@ public void putPipeline( })); } - public void validatePipelineRequest(PutPipelineRequest request, NodesInfoResponse nodeInfos) throws Exception { + public void validatePipelineRequest(ProjectId projectId, PutPipelineRequest request, NodesInfoResponse nodeInfos) throws Exception { final Map config = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2(); Map ingestInfos = new HashMap<>(); for (NodeInfo nodeInfo : nodeInfos.getNodes()) { ingestInfos.put(nodeInfo.getNode(), nodeInfo.getInfo(IngestInfo.class)); } - validatePipeline(ingestInfos, request.getId(), config); + validatePipeline(ingestInfos, projectId, request.getId(), config); } public static boolean isNoOpPipelineUpdate(ProjectMetadata metadata, PutPipelineRequest request) { @@ -729,8 +733,12 @@ public IngestMetadata execute(IngestMetadata currentIngestMetadata, Collection ingestInfos, String pipelineId, Map pipelineConfig) - throws Exception { + void validatePipeline( + Map ingestInfos, + ProjectId projectId, + String pipelineId, + Map pipelineConfig + ) throws Exception { if (ingestInfos.isEmpty()) { throw new IllegalStateException("Ingest info is empty"); } @@ -746,7 +754,7 @@ void validatePipeline(Map ingestInfos, String pipelin deprecationLogger.critical(DeprecationCategory.API, "pipeline_name_special_chars", e.getMessage()); } - Pipeline pipeline = Pipeline.create(pipelineId, pipelineConfig, processorFactories, scriptService); + Pipeline pipeline = Pipeline.create(pipelineId, pipelineConfig, processorFactories, scriptService, projectId); List exceptions = new ArrayList<>(); for (Processor processor : pipeline.flattenAllProcessors()) { @@ -1394,7 +1402,8 @@ synchronized void innerUpdatePipelines(ProjectId projectId, IngestMetadata newIn newConfiguration.getId(), newConfiguration.getConfig(false), processorFactories, - scriptService + scriptService, + projectId ); newPipelines.put(newConfiguration.getId(), new PipelineHolder(newConfiguration, newPipeline)); @@ -1523,7 +1532,7 @@ public

Collection getPipelineWithProcessorType( public synchronized void reloadPipeline(ProjectId projectId, String id) throws Exception { var originalPipelines = this.pipelines.getOrDefault(projectId, ImmutableOpenMap.of()); PipelineHolder holder = originalPipelines.get(id); - Pipeline updatedPipeline = Pipeline.create(id, holder.configuration.getConfig(false), processorFactories, scriptService); + Pipeline updatedPipeline = Pipeline.create(id, holder.configuration.getConfig(false), processorFactories, scriptService, projectId); ImmutableOpenMap updatedPipelines = ImmutableOpenMap.builder(originalPipelines) .fPut(id, new PipelineHolder(holder.configuration, updatedPipeline)) .build(); diff --git a/server/src/main/java/org/elasticsearch/ingest/Pipeline.java b/server/src/main/java/org/elasticsearch/ingest/Pipeline.java index a8e8fbb5d3217..b0747ddc20589 100644 --- a/server/src/main/java/org/elasticsearch/ingest/Pipeline.java +++ b/server/src/main/java/org/elasticsearch/ingest/Pipeline.java @@ -10,6 +10,7 @@ package org.elasticsearch.ingest; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.core.Nullable; import org.elasticsearch.script.ScriptService; @@ -89,19 +90,26 @@ public static Pipeline create( String id, Map config, Map processorFactories, - ScriptService scriptService + ScriptService scriptService, + ProjectId projectId ) throws Exception { String description = ConfigurationUtils.readOptionalStringProperty(null, null, config, DESCRIPTION_KEY); Integer version = ConfigurationUtils.readIntProperty(null, null, config, VERSION_KEY, null); Map metadata = ConfigurationUtils.readOptionalMap(null, null, config, META_KEY); Boolean deprecated = ConfigurationUtils.readOptionalBooleanProperty(null, null, config, DEPRECATED_KEY); List> processorConfigs = ConfigurationUtils.readList(null, null, config, PROCESSORS_KEY); - List processors = ConfigurationUtils.readProcessorConfigs(processorConfigs, scriptService, processorFactories); + List processors = ConfigurationUtils.readProcessorConfigs( + processorConfigs, + scriptService, + processorFactories, + projectId + ); List> onFailureProcessorConfigs = ConfigurationUtils.readOptionalList(null, null, config, ON_FAILURE_KEY); List onFailureProcessors = ConfigurationUtils.readProcessorConfigs( onFailureProcessorConfigs, scriptService, - processorFactories + processorFactories, + projectId ); if (config.isEmpty() == false) { throw new ElasticsearchParseException( diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java b/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java index 5a4933481227a..a8a2a5dfba115 100644 --- a/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.script.TemplateScript; import java.util.Map; @@ -93,7 +94,8 @@ public PipelineProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { TemplateScript.Factory pipelineTemplate = ConfigurationUtils.readTemplateProperty( TYPE, diff --git a/server/src/main/java/org/elasticsearch/ingest/Processor.java b/server/src/main/java/org/elasticsearch/ingest/Processor.java index e168f59040d20..4cb5e56c04c1d 100644 --- a/server/src/main/java/org/elasticsearch/ingest/Processor.java +++ b/server/src/main/java/org/elasticsearch/ingest/Processor.java @@ -10,6 +10,7 @@ package org.elasticsearch.ingest; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.env.Environment; import org.elasticsearch.grok.MatcherWatchdog; @@ -84,12 +85,12 @@ default boolean isAsync() { * * Override this method to perform additional post-construction validation that should be performed at the rest/transport level. * If there's an issue with the processor, then indicate that by throwing an exception. See - * {@link IngestService#validatePipeline(Map, String, Map)}} for the call site where there is invoked in a try/catch. + * {@link IngestService#validatePipeline(Map, ProjectId, String, Map)}} for the call site where there is invoked in a try/catch. * * An example of where this would be needed is a processor that interacts with external state like the license state -- it may * be okay to create that processor on day 1 with license state A, but later illegal to create a similar processor on day 2 with * state B. We want to reject put requests on day 2 (at the rest/transport level), but still allow for restarting nodes in the - * cluster (so we can't throw exceptions from {@link Factory#create(Map, String, String, Map)}). + * cluster (so we can't throw exceptions from {@link Factory#create(Map, String, String, Map, ProjectId)}). */ default void extraValidation() throws Exception {} @@ -100,15 +101,22 @@ interface Factory { /** * Creates a processor based on the specified map of maps config. - * @param processorFactories Other processors which may be created inside this processor + * + * @param processorFactories Other processors which may be created inside this processor * @param tag The tag for the processor * @param description A short description of what this processor does * @param config The configuration for the processor + * @param projectId The project for which the processor is created * * Note: Implementations are responsible for removing the used configuration keys, so that after */ - Processor create(Map processorFactories, String tag, String description, Map config) - throws Exception; + Processor create( + Map processorFactories, + String tag, + String description, + Map config, + ProjectId projectId + ) throws Exception; } /** diff --git a/server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java b/server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java index e2d18174a6bf8..c4fa1358d8219 100644 --- a/server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java @@ -57,7 +57,8 @@ private Map getPipelineSubstitutions( pipelineId, entry.getValue(), ingestService.getProcessorFactories(), - ingestService.getScriptService() + ingestService.getScriptService(), + ingestService.getProjectResolver().getProjectId() ); parsedPipelineSubstitutions.put(pipelineId, pipeline); } diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java index c9d98300df6f3..0d33ba2695f29 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java @@ -261,7 +261,7 @@ public void testExecuteItemWithFailure() throws Exception { public void testDropDocument() throws Exception { TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field", "value")); - Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of()); + Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of(), null); Pipeline pipeline = new Pipeline("_id", "_description", version, null, new CompoundProcessor(processor1, processor2)); CountDownLatch latch = new CountDownLatch(1); @@ -281,7 +281,7 @@ public void testDropDocument() throws Exception { public void testDropDocumentVerbose() throws Exception { TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field", "value")); - Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of()); + Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of(), null); Pipeline pipeline = new Pipeline("_id", "_description", version, null, new CompoundProcessor(processor1, processor2)); CountDownLatch latch = new CountDownLatch(1); @@ -304,7 +304,7 @@ public void testDropDocumentVerbose() throws Exception { public void testDropDocumentVerboseExtraProcessor() throws Exception { TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field1", "value")); - Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of()); + Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of(), null); TestProcessor processor3 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field2", "value")); Pipeline pipeline = new Pipeline("_id", "_description", version, null, new CompoundProcessor(processor1, processor2, processor3)); diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index 575c3e87dfcd7..a71a2f1c6e267 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -60,7 +60,7 @@ public void init() throws IOException { Pipeline pipeline = new Pipeline(SIMULATED_PIPELINE_ID, null, null, null, pipelineCompoundProcessor); Map registry = Collections.singletonMap( "mock_processor", - (factories, tag, description, config) -> processor + (factories, tag, description, config, projectId) -> processor ); ingestService = mock(IngestService.class); when(ingestService.getPipeline(any(), eq(SIMULATED_PIPELINE_ID))).thenReturn(pipeline); @@ -189,7 +189,9 @@ public void testParseWithProvidedPipeline() throws Exception { requestContent.put(Fields.PIPELINE, pipelineConfig); + var projectId = randomProjectIdOrDefault(); SimulatePipelineRequest.Parsed actualRequest = SimulatePipelineRequest.parse( + projectId, requestContent, false, ingestService, @@ -256,6 +258,7 @@ public void testNonExistentPipelineId() { } public void testNotValidDocs() { + var projectId = randomProjectIdOrDefault(); Map requestContent = new HashMap<>(); List> docs = new ArrayList<>(); Map pipelineConfig = new HashMap<>(); @@ -265,7 +268,7 @@ public void testNotValidDocs() { requestContent.put(Fields.PIPELINE, pipelineConfig); Exception e1 = expectThrows( IllegalArgumentException.class, - () -> SimulatePipelineRequest.parse(requestContent, false, ingestService, RestApiVersion.current()) + () -> SimulatePipelineRequest.parse(projectId, requestContent, false, ingestService, RestApiVersion.current()) ); assertThat(e1.getMessage(), equalTo("must specify at least one document in [docs]")); @@ -276,7 +279,7 @@ public void testNotValidDocs() { requestContent.put(Fields.PIPELINE, pipelineConfig); Exception e2 = expectThrows( IllegalArgumentException.class, - () -> SimulatePipelineRequest.parse(requestContent, false, ingestService, RestApiVersion.current()) + () -> SimulatePipelineRequest.parse(projectId, requestContent, false, ingestService, RestApiVersion.current()) ); assertThat(e2.getMessage(), equalTo("malformed [docs] section, should include an inner object")); @@ -285,7 +288,7 @@ public void testNotValidDocs() { requestContent.put(Fields.PIPELINE, pipelineConfig); Exception e3 = expectThrows( ElasticsearchParseException.class, - () -> SimulatePipelineRequest.parse(requestContent, false, ingestService, RestApiVersion.current()) + () -> SimulatePipelineRequest.parse(projectId, requestContent, false, ingestService, RestApiVersion.current()) ); assertThat(e3.getMessage(), containsString("required property is missing")); } @@ -358,7 +361,9 @@ public void testIngestPipelineWithDocumentsWithType() throws Exception { pipelineConfig.put("on_failure", onFailureProcessors); } requestContent.put(Fields.PIPELINE, pipelineConfig); + var projectId = randomProjectIdOrDefault(); SimulatePipelineRequest.Parsed actualRequest = SimulatePipelineRequest.parse( + projectId, requestContent, false, ingestService, diff --git a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java index 0e8c7e0857251..3e282f133abe9 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.TemplateScript; import org.elasticsearch.test.ESTestCase; @@ -40,6 +41,7 @@ public class ConfigurationUtilsTests extends ESTestCase { private final ScriptService scriptService = mock(ScriptService.class); + private final ProjectId projectId = randomProjectIdOrDefault(); private Map config; @Before @@ -148,7 +150,7 @@ public void testReadProcessors() throws Exception { Processor processor = mock(Processor.class); Map registry = Collections.singletonMap( "test_processor", - (factories, tag, description, config) -> processor + (factories, tag, description, config, projectId) -> processor ); List> config = new ArrayList<>(); @@ -156,7 +158,7 @@ public void testReadProcessors() throws Exception { config.add(Collections.singletonMap("test_processor", emptyConfig)); config.add(Collections.singletonMap("test_processor", emptyConfig)); - List result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry); + List result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry, projectId); assertThat(result.size(), equalTo(2)); assertThat(result.get(0), sameInstance(processor)); assertThat(result.get(1), sameInstance(processor)); @@ -169,7 +171,7 @@ public void testReadProcessors() throws Exception { config.add(Collections.singletonMap("unknown_processor", unknownTaggedConfig)); ElasticsearchParseException e = expectThrows( ElasticsearchParseException.class, - () -> ConfigurationUtils.readProcessorConfigs(config, scriptService, registry) + () -> ConfigurationUtils.readProcessorConfigs(config, scriptService, registry, projectId) ); assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]")); assertThat(e.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_unknown"))); @@ -186,7 +188,7 @@ public void testReadProcessors() throws Exception { config2.add(Collections.singletonMap("second_unknown_processor", secondUnknownTaggedConfig)); e = expectThrows( ElasticsearchParseException.class, - () -> ConfigurationUtils.readProcessorConfigs(config2, scriptService, registry) + () -> ConfigurationUtils.readProcessorConfigs(config2, scriptService, registry, projectId) ); assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]")); assertThat(e.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_unknown"))); @@ -205,7 +207,7 @@ public void testReadProcessors() throws Exception { config3.add(Collections.singletonMap("test_processor", null)); ElasticsearchParseException e3 = expectThrows( ElasticsearchParseException.class, - () -> ConfigurationUtils.readProcessorConfigs(config3, scriptService, registry) + () -> ConfigurationUtils.readProcessorConfigs(config3, scriptService, registry, projectId) ); assertThat(e3.getMetadata("es.processor_type"), equalTo(Collections.singletonList("test_processor"))); assertThat(e3.getMessage(), equalTo("processor config cannot be [null]")); @@ -213,15 +215,18 @@ public void testReadProcessors() throws Exception { public void testReadProcessorNullDescription() throws Exception { Processor processor = new TestProcessor("tag", "type", null, (ingestDocument) -> {}); - Map registry = Collections.singletonMap("test_processor", (factories, tag, description, config) -> { - assertNull(description); - return processor; - }); + Map registry = Collections.singletonMap( + "test_processor", + (factories, tag, description, config, projectId) -> { + assertNull(description); + return processor; + } + ); List> config = new ArrayList<>(); Map emptyConfig = Collections.emptyMap(); config.add(Collections.singletonMap("test_processor", emptyConfig)); - List result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry); + List result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry, projectId); assertThat(result.size(), equalTo(1)); assertThat(result.get(0), sameInstance(processor)); } @@ -229,40 +234,46 @@ public void testReadProcessorNullDescription() throws Exception { public void testReadProcessorDescription() throws Exception { String testDescription = randomAlphaOfLengthBetween(10, 20); Processor processor = new TestProcessor("tag", "type", testDescription, (ingestDocument) -> {}); - Map registry = Collections.singletonMap("test_processor", (factories, tag, description, config) -> { - assertThat(description, equalTo(processor.getDescription())); - return processor; - }); + Map registry = Collections.singletonMap( + "test_processor", + (factories, tag, description, config, projectId) -> { + assertThat(description, equalTo(processor.getDescription())); + return processor; + } + ); List> config = new ArrayList<>(); Map processorConfig = new HashMap<>(); processorConfig.put(ConfigurationUtils.DESCRIPTION_KEY, testDescription); config.add(Collections.singletonMap("test_processor", processorConfig)); - List result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry); + List result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry, projectId); assertThat(result.size(), equalTo(1)); assertThat(result.get(0), sameInstance(processor)); } public void testReadProcessorFromObjectOrMap() throws Exception { Processor processor = mock(Processor.class); - Map registry = Collections.singletonMap("script", (processorFactories, tag, description, config) -> { - config.clear(); - return processor; - }); + Map registry = Collections.singletonMap( + "script", + (processorFactories, tag, description, config, projectId) -> { + config.clear(); + return processor; + } + ); Object emptyConfig = Collections.emptyMap(); - Processor processor1 = ConfigurationUtils.readProcessor(registry, scriptService, "script", emptyConfig); + Processor processor1 = ConfigurationUtils.readProcessor(registry, scriptService, "script", emptyConfig, projectId); assertThat(processor1, sameInstance(processor)); Object inlineScript = "test_script"; - Processor processor2 = ConfigurationUtils.readProcessor(registry, scriptService, "script", inlineScript); + Processor processor2 = ConfigurationUtils.readProcessor(registry, scriptService, "script", inlineScript, projectId); assertThat(processor2, sameInstance(processor)); Object invalidConfig = 12L; ElasticsearchParseException ex = expectThrows( ElasticsearchParseException.class, - () -> ConfigurationUtils.readProcessor(registry, scriptService, "unknown_processor", invalidConfig) + () -> ConfigurationUtils.readProcessor(registry, scriptService, "unknown_processor", invalidConfig, projectId) ); assertThat(ex.getMessage(), equalTo("property isn't a map, but of type [" + invalidConfig.getClass().getName() + "]")); } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 36ce4292c7d6d..7c5fe13d9a3cf 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -135,7 +135,7 @@ public class IngestServiceTests extends ESTestCase { private static final IngestPlugin DUMMY_PLUGIN = new IngestPlugin() { @Override public Map getProcessors(Processor.Parameters parameters) { - return Map.of("foo", (factories, tag, description, config) -> null); + return Map.of("foo", (factories, tag, description, config, projectId) -> null); } }; @@ -347,11 +347,11 @@ public void testInnerUpdatePipelines() { public void testInnerUpdatePipelinesValidation() { Map processors = new HashMap<>(); - processors.put("fail_validation", (factories, tag, description, config) -> { + processors.put("fail_validation", (factories, tag, description, config, projectId) -> { // ordinary validation issues happen at processor construction time throw newConfigurationException("fail_validation", tag, "no_property_name", "validation failure reason"); }); - processors.put("fail_extra_validation", (factories, tag, description, config) -> { + processors.put("fail_extra_validation", (factories, tag, description, config, projectId) -> { // 'extra validation' issues happen post- processor construction time return new FakeProcessor("fail_extra_validation", tag, description, ingestDocument -> {}) { @Override @@ -444,16 +444,17 @@ public void testValidateNoIngestInfo() throws Exception { PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """ {"processors": [{"set" : {"field": "_field", "value": "_value"}}]}"""); + var projectId = randomProjectIdOrDefault(); var pipelineConfig = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2(); Exception e = expectThrows( IllegalStateException.class, - () -> ingestService.validatePipeline(Map.of(), putRequest.getId(), pipelineConfig) + () -> ingestService.validatePipeline(Map.of(), projectId, putRequest.getId(), pipelineConfig) ); assertEquals("Ingest info is empty", e.getMessage()); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("_node_id", buildNewFakeTransportAddress(), Map.of(), Set.of()); IngestInfo ingestInfo = new IngestInfo(List.of(new ProcessorInfo("set"))); - ingestService.validatePipeline(Map.of(discoveryNode, ingestInfo), putRequest.getId(), pipelineConfig); + ingestService.validatePipeline(Map.of(discoveryNode, ingestInfo), projectId, putRequest.getId(), pipelineConfig); } public void testValidateNotInUse() { @@ -618,7 +619,7 @@ public void testReloadPipeline() throws Exception { boolean[] externalProperty = new boolean[] { false }; Map processorFactories = new HashMap<>(); - processorFactories.put("set", (factories, tag, description, config) -> { + processorFactories.put("set", (factories, tag, description, config, projectId) -> { String field = (String) config.remove("field"); String value = (String) config.remove("value"); if (externalProperty[0]) { @@ -685,7 +686,7 @@ public void testGetProcessorsInPipelineComplexConditional() throws Exception { ); Map processors = new HashMap<>(); - processors.put("complexSet", (factories, tag, description, config) -> { + processors.put("complexSet", (factories, tag, description, config, projectId) -> { String field = (String) config.remove("field"); String value = (String) config.remove("value"); @@ -1017,6 +1018,7 @@ public void testGetPipelines() { public void testValidateProcessorTypeOnAllNodes() throws Exception { IngestService ingestService = createWithProcessors(); + var projectId = randomProjectIdOrDefault(); PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """ { "processors": [ @@ -1045,7 +1047,7 @@ public void testValidateProcessorTypeOnAllNodes() throws Exception { ElasticsearchParseException e = expectThrows( ElasticsearchParseException.class, - () -> ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig) + () -> ingestService.validatePipeline(ingestInfos, projectId, putRequest.getId(), pipelineConfig) ); assertEquals("Processor type [remove] is not installed on node [" + node2 + "]", e.getMessage()); assertEquals("remove", e.getMetadata("es.processor_type").get(0)); @@ -1053,14 +1055,15 @@ public void testValidateProcessorTypeOnAllNodes() throws Exception { var pipelineConfig2 = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2(); ingestInfos.put(node2, new IngestInfo(List.of(new ProcessorInfo("set"), new ProcessorInfo("remove")))); - ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig2); + ingestService.validatePipeline(ingestInfos, projectId, putRequest.getId(), pipelineConfig2); } public void testValidateConfigurationExceptions() { - IngestService ingestService = createWithProcessors(Map.of("fail_validation", (factories, tag, description, config) -> { + IngestService ingestService = createWithProcessors(Map.of("fail_validation", (factories, tag, description, config, projectId) -> { // ordinary validation issues happen at processor construction time throw newConfigurationException("fail_validation", tag, "no_property_name", "validation failure reason"); })); + var projectId = randomProjectIdOrDefault(); PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """ { "processors": [ @@ -1079,22 +1082,30 @@ public void testValidateConfigurationExceptions() { ElasticsearchParseException e = expectThrows( ElasticsearchParseException.class, - () -> ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig) + () -> ingestService.validatePipeline(ingestInfos, projectId, putRequest.getId(), pipelineConfig) ); assertEquals("[no_property_name] validation failure reason", e.getMessage()); assertEquals("fail_validation", e.getMetadata("es.processor_type").get(0)); } public void testValidateExtraValidationConfigurationExceptions() { - IngestService ingestService = createWithProcessors(Map.of("fail_extra_validation", (factories, tag, description, config) -> { - // 'extra validation' issues happen post- processor construction time - return new FakeProcessor("fail_extra_validation", tag, description, ingestDocument -> {}) { - @Override - public void extraValidation() throws Exception { - throw newConfigurationException("fail_extra_validation", tag, "no_property_name", "extra validation failure reason"); - } - }; - })); + IngestService ingestService = createWithProcessors( + Map.of("fail_extra_validation", (factories, tag, description, config, projectId) -> { + // 'extra validation' issues happen post- processor construction time + return new FakeProcessor("fail_extra_validation", tag, description, ingestDocument -> {}) { + @Override + public void extraValidation() throws Exception { + throw newConfigurationException( + "fail_extra_validation", + tag, + "no_property_name", + "extra validation failure reason" + ); + } + }; + }) + ); + var projectId = randomProjectIdOrDefault(); PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """ { "processors": [ @@ -1113,13 +1124,14 @@ public void extraValidation() throws Exception { ElasticsearchParseException e = expectThrows( ElasticsearchParseException.class, - () -> ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig) + () -> ingestService.validatePipeline(ingestInfos, projectId, putRequest.getId(), pipelineConfig) ); assertEquals("[no_property_name] extra validation failure reason", e.getMessage()); assertEquals("fail_extra_validation", e.getMetadata("es.processor_type").get(0)); } public void testValidatePipelineName() throws Exception { + var projectId = randomProjectIdOrDefault(); IngestService ingestService = createWithProcessors(); for (Character badChar : List.of('\\', '/', '*', '?', '"', '<', '>', '|', ' ', ',')) { PutPipelineRequest putRequest = new PutPipelineRequest( @@ -1135,7 +1147,7 @@ public void testValidatePipelineName() throws Exception { Map ingestInfos = new HashMap<>(); ingestInfos.put(node1, new IngestInfo(List.of(new ProcessorInfo("set")))); final String name = randomAlphaOfLength(5) + badChar + randomAlphaOfLength(5); - ingestService.validatePipeline(ingestInfos, name, pipelineConfig); + ingestService.validatePipeline(ingestInfos, projectId, name, pipelineConfig); assertCriticalWarnings( "Pipeline name [" + name @@ -1147,7 +1159,7 @@ public void testValidatePipelineName() throws Exception { public void testExecuteIndexPipelineExistsButFailedParsing() { IngestService ingestService = createWithProcessors( - Map.of("mock", (factories, tag, description, config) -> new AbstractProcessor("mock", "description") { + Map.of("mock", (factories, tag, description, config, projectId) -> new AbstractProcessor("mock", "description") { @Override public IngestDocument execute(IngestDocument ingestDocument) { throw new IllegalStateException("error"); @@ -1213,7 +1225,7 @@ public String getType() { public void testExecuteBulkPipelineDoesNotExist() { IngestService ingestService = createWithProcessors( - Map.of("mock", (factories, tag, description, config) -> mockCompoundProcessor()) + Map.of("mock", (factories, tag, description, config, projectId) -> mockCompoundProcessor()) ); PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}"); @@ -1267,7 +1279,7 @@ public void testExecuteBulkPipelineDoesNotExist() { public void testExecuteSuccess() { IngestService ingestService = createWithProcessors( - Map.of("mock", (factories, tag, description, config) -> mockCompoundProcessor()) + Map.of("mock", (factories, tag, description, config, projectId) -> mockCompoundProcessor()) ); PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}"); var projectId = randomProjectIdOrDefault(); @@ -1304,7 +1316,7 @@ public void testDynamicTemplates() throws Exception { IngestService ingestService = createWithProcessors( Map.of( "set", - (factories, tag, description, config) -> new FakeProcessor( + (factories, tag, description, config, projectId) -> new FakeProcessor( "set", "", "", @@ -1380,7 +1392,7 @@ public void testExecuteEmptyPipeline() throws Exception { public void testExecutePropagateAllMetadataUpdates() throws Exception { final CompoundProcessor processor = mockCompoundProcessor(); - IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> processor)); + IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config, projectId) -> processor)); PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}"); var projectId = randomProjectIdOrDefault(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) @@ -1454,9 +1466,9 @@ public void testExecuteFailure() throws Exception { IngestService ingestService = createWithProcessors( Map.of( "mock", - (factories, tag, description, config) -> processor, + (factories, tag, description, config, projectId) -> processor, "set", - (factories, tag, description, config) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail()) + (factories, tag, description, config, projectId) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail()) ) ); PutPipelineRequest putRequest1 = putJsonPipelineRequest("_id1", "{\"processors\": [{\"mock\" : {}}]}"); @@ -1526,7 +1538,9 @@ public void testExecuteSuccessWithOnFailure() throws Exception { List.of(processor), List.of(new CompoundProcessor(onFailureProcessor)) ); - IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> compoundProcessor)); + IngestService ingestService = createWithProcessors( + Map.of("mock", (factories, tag, description, config, projectId) -> compoundProcessor) + ); PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}"); var projectId = randomProjectIdOrDefault(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) @@ -1572,7 +1586,9 @@ public void testExecuteFailureWithNestedOnFailure() throws Exception { List.of(processor), List.of(new CompoundProcessor(false, processors, onFailureProcessors)) ); - IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> compoundProcessor)); + IngestService ingestService = createWithProcessors( + Map.of("mock", (factories, tag, description, config, projectId) -> compoundProcessor) + ); PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}"); var projectId = randomProjectIdOrDefault(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) @@ -1650,7 +1666,7 @@ public void testBulkRequestExecutionWithFailures() throws Exception { handler.accept(null, error); return null; }).when(processor).execute(any(), any()); - IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> processor)); + IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config, projectId) -> processor)); PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}"); var projectId = randomProjectIdOrDefault(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) @@ -1691,9 +1707,9 @@ public void testExecuteFailureRedirection() throws Exception { IngestService ingestService = createWithProcessors( Map.of( "mock", - (factories, tag, description, config) -> processor, + (factories, tag, description, config, projectId) -> processor, "set", - (factories, tag, description, config) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail()) + (factories, tag, description, config, projectId) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail()) ) ); PutPipelineRequest putRequest1 = putJsonPipelineRequest("_id1", "{\"processors\": [{\"mock\" : {}}]}"); @@ -1742,9 +1758,9 @@ public void testExecuteFailureStatusOnFailureWithoutRedirection() throws Excepti IngestService ingestService = createWithProcessors( Map.of( "mock", - (factories, tag, description, config) -> processor, + (factories, tag, description, config, projectId) -> processor, "set", - (factories, tag, description, config) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail()) + (factories, tag, description, config, projectId) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail()) ) ); PutPipelineRequest putRequest1 = putJsonPipelineRequest("_id1", "{\"processors\": [{\"mock\" : {}}]}"); @@ -1799,7 +1815,9 @@ public void testExecuteFailureRedirectionWithNestedOnFailure() throws Exception List.of(processor), List.of(new CompoundProcessor(false, processors, onFailureProcessors)) ); - IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> compoundProcessor)); + IngestService ingestService = createWithProcessors( + Map.of("mock", (factories, tag, description, config, projectId) -> compoundProcessor) + ); PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}"); var projectId = randomProjectIdOrDefault(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) @@ -1875,7 +1893,7 @@ public void testBulkRequestExecutionWithRedirectedFailures() throws Exception { handler.accept(null, error); return null; }).when(processor).execute(any(), any()); - IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> processor)); + IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config, projectId) -> processor)); PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}"); var projectId = randomProjectIdOrDefault(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) @@ -1939,7 +1957,7 @@ public void testBulkRequestExecution() throws Exception { return null; }).when(processor).execute(any(), any()); Map map = Maps.newMapWithExpectedSize(2); - map.put("mock", (factories, tag, description, config) -> processor); + map.put("mock", (factories, tag, description, config, projectId) -> processor); IngestService ingestService = createWithProcessors(map); PutPipelineRequest putRequest = putJsonPipelineRequest("_id", """ @@ -2007,14 +2025,20 @@ public void testIngestAndPipelineStats() throws Exception { IngestService ingestService = createWithProcessors( Map.of( "pipeline", - (factories, tag, description, config) -> new PipelineProcessor(tag, description, (params) -> new TemplateScript(params) { - @Override - public String execute() { - return "_id3"; - } // this pipeline processor will always execute the '_id3' processor - }, false, pipelineIngestService), + (factories, tag, description, config, projectId) -> new PipelineProcessor( + tag, + description, + (params) -> new TemplateScript(params) { + @Override + public String execute() { + return "_id3"; + } // this pipeline processor will always execute the '_id3' processor + }, + false, + pipelineIngestService + ), "mock", - (factories, tag, description, config) -> processor + (factories, tag, description, config, projectId) -> processor ) ); @@ -2116,8 +2140,8 @@ public void testStats() throws Exception { return null; }).when(processorFailure).execute(any(IngestDocument.class), any()); Map map = Maps.newMapWithExpectedSize(2); - map.put("mock", (factories, tag, description, config) -> processor); - map.put("failure-mock", (factories, tag, description, config) -> processorFailure); + map.put("mock", (factories, tag, description, config, projectId) -> processor); + map.put("failure-mock", (factories, tag, description, config, projectId) -> processorFailure); map.put("drop", new DropProcessor.Factory()); IngestService ingestService = createWithProcessors(map); @@ -2324,7 +2348,7 @@ public void testStatName() { public void testExecuteWithDrop() { Map factories = new HashMap<>(); factories.put("drop", new DropProcessor.Factory()); - factories.put("mock", (processorFactories, tag, description, config) -> new Processor() { + factories.put("mock", (processorFactories, tag, description, config, projectId) -> new Processor() { @Override public IngestDocument execute(final IngestDocument ingestDocument) { throw new AssertionError("Document should have been dropped but reached this processor"); @@ -2397,7 +2421,7 @@ public void testIngestClusterStateListeners_orderOfExecution() { IngestPlugin testPlugin = new IngestPlugin() { @Override public Map getProcessors(Processor.Parameters parameters) { - return Map.of("test", (factories, tag, description, config) -> { + return Map.of("test", (factories, tag, description, config, projectId) -> { assertThat(counter.compareAndSet(1, 2), is(true)); return new FakeProcessor("test", tag, description, ingestDocument -> {}); }); @@ -2438,7 +2462,7 @@ public void testCBORParsing() throws Exception { AtomicReference reference = new AtomicReference<>(); Consumer executor = doc -> reference.set(doc.getFieldValueAsBytes("data")); final IngestService ingestService = createWithProcessors( - Map.of("foo", (factories, tag, description, config) -> new FakeProcessor("foo", tag, description, executor)) + Map.of("foo", (factories, tag, description, config, projectId) -> new FakeProcessor("foo", tag, description, executor)) ); var projectId = randomProjectIdOrDefault(); @@ -2481,9 +2505,9 @@ public void testSetsRawTimestamp() { IngestService ingestService = createWithProcessors( Map.of( "mock", - (factories, tag, description, config) -> mockCompoundProcessor(), + (factories, tag, description, config, projectId) -> mockCompoundProcessor(), "set", - (factories, tag, description, config) -> new FakeProcessor( + (factories, tag, description, config, projectId) -> new FakeProcessor( "set", tag, description, @@ -3206,12 +3230,12 @@ private IngestDocument eqIndexTypeId(final long version, final VersionType versi private static IngestService createWithProcessors() { Map processors = new HashMap<>(); - processors.put("set", (factories, tag, description, config) -> { + processors.put("set", (factories, tag, description, config, projectId) -> { String field = (String) config.remove("field"); String value = (String) config.remove("value"); return new FakeProcessor("set", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value)); }); - processors.put("remove", (factories, tag, description, config) -> { + processors.put("remove", (factories, tag, description, config, projectId) -> { String field = (String) config.remove("field"); return new WrappingProcessorImpl("remove", tag, description, (ingestDocument -> ingestDocument.removeField(field))) { }; diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java index 39a07d8b5f4de..21bdbe2972a22 100644 --- a/server/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java @@ -10,6 +10,7 @@ package org.elasticsearch.ingest; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.core.Tuple; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; @@ -26,6 +27,7 @@ public class PipelineFactoryTests extends ESTestCase { + private final ProjectId projectId = randomProjectIdOrDefault(); private final Integer version = randomBoolean() ? randomInt() : null; private final String versionString = version != null ? Integer.toString(version) : null; private final ScriptService scriptService = mock(ScriptService.class); @@ -45,7 +47,7 @@ public void testCreate() throws Exception { pipelineConfig.put(Pipeline.DEPRECATED_KEY, deprecated); pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig0), Map.of("test", processorConfig1))); Map processorRegistry = Map.of("test", new TestProcessor.Factory()); - Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService); + Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null); assertThat(pipeline.getId(), equalTo("_id")); assertThat(pipeline.getDescription(), equalTo("_description")); assertThat(pipeline.getVersion(), equalTo(version)); @@ -65,7 +67,7 @@ public void testCreateWithNoProcessorsField() throws Exception { pipelineConfig.put(Pipeline.META_KEY, metadata); } try { - Pipeline.create("_id", pipelineConfig, Map.of(), scriptService); + Pipeline.create("_id", pipelineConfig, Map.of(), scriptService, null); fail("should fail, missing required [processors] field"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("[processors] required property is missing")); @@ -80,7 +82,7 @@ public void testCreateWithEmptyProcessorsField() throws Exception { pipelineConfig.put(Pipeline.META_KEY, metadata); } pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of()); - Pipeline pipeline = Pipeline.create("_id", pipelineConfig, null, scriptService); + Pipeline pipeline = Pipeline.create("_id", pipelineConfig, null, scriptService, null); assertThat(pipeline.getId(), equalTo("_id")); assertThat(pipeline.getDescription(), equalTo("_description")); assertThat(pipeline.getVersion(), equalTo(version)); @@ -98,7 +100,7 @@ public void testCreateWithPipelineOnFailure() throws Exception { pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig))); pipelineConfig.put(Pipeline.ON_FAILURE_KEY, List.of(Map.of("test", processorConfig))); Map processorRegistry = Map.of("test", new TestProcessor.Factory()); - Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService); + Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null); assertThat(pipeline.getId(), equalTo("_id")); assertThat(pipeline.getDescription(), equalTo("_description")); assertThat(pipeline.getVersion(), equalTo(version)); @@ -121,7 +123,7 @@ public void testCreateWithPipelineEmptyOnFailure() throws Exception { Map processorRegistry = Map.of("test", new TestProcessor.Factory()); Exception e = expectThrows( ElasticsearchParseException.class, - () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService) + () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null) ); assertThat(e.getMessage(), equalTo("pipeline [_id] cannot have an empty on_failure option defined")); } @@ -139,7 +141,7 @@ public void testCreateWithPipelineEmptyOnFailureInProcessor() throws Exception { Map processorRegistry = Map.of("test", new TestProcessor.Factory()); Exception e = expectThrows( ElasticsearchParseException.class, - () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService) + () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null) ); assertThat(e.getMessage(), equalTo("[on_failure] processors list cannot be empty")); } @@ -157,7 +159,7 @@ public void testCreateWithPipelineIgnoreFailure() throws Exception { } pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig))); - Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService); + Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null); assertThat(pipeline.getId(), equalTo("_id")); assertThat(pipeline.getDescription(), equalTo("_description")); assertThat(pipeline.getVersion(), equalTo(version)); @@ -182,7 +184,7 @@ public void testCreateUnusedProcessorOptions() throws Exception { Map processorRegistry = Map.of("test", new TestProcessor.Factory()); Exception e = expectThrows( ElasticsearchParseException.class, - () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService) + () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null) ); assertThat(e.getMessage(), equalTo("processor [test] doesn't support one or more provided configuration parameters [unused]")); } @@ -199,7 +201,7 @@ public void testCreateProcessorsWithOnFailureProperties() throws Exception { } pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig))); Map processorRegistry = Map.of("test", new TestProcessor.Factory()); - Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService); + Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null); assertThat(pipeline.getId(), equalTo("_id")); assertThat(pipeline.getDescription(), equalTo("_description")); assertThat(pipeline.getVersion(), equalTo(version)); diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java index 43a7af54b4ada..cf11c29978c7a 100644 --- a/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java @@ -61,7 +61,7 @@ public String getDescription() { PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); Map config = new HashMap<>(); config.put("name", pipelineId); - factory.create(Map.of(), null, null, config).execute(testIngestDocument, (result, e) -> {}); + factory.create(Map.of(), null, null, config, null).execute(testIngestDocument, (result, e) -> {}); assertIngestDocument(testIngestDocument, safeGet(invoked)); } @@ -72,7 +72,7 @@ public void testThrowsOnMissingPipeline() throws Exception { Map config = new HashMap<>(); config.put("name", "missingPipelineId"); IllegalStateException[] e = new IllegalStateException[1]; - factory.create(Map.of(), null, null, config).execute(testIngestDocument, (result, e1) -> e[0] = (IllegalStateException) e1); + factory.create(Map.of(), null, null, config, null).execute(testIngestDocument, (result, e1) -> e[0] = (IllegalStateException) e1); assertEquals("Pipeline processor configured for non-existent pipeline [missingPipelineId]", e[0].getMessage()); } @@ -86,7 +86,7 @@ public void testIgnoreMissingPipeline() throws Exception { var r = new IngestDocument[1]; var e = new Exception[1]; - var processor = factory.create(Map.of(), null, null, config); + var processor = factory.create(Map.of(), null, null, config, null); processor.execute(testIngestDocument, (result, e1) -> { r[0] = result; e[0] = e1; @@ -108,7 +108,7 @@ public void testThrowsOnRecursivePipelineInvocations() throws Exception { null, null, null, - new CompoundProcessor(factory.create(Map.of(), null, null, outerConfig)) + new CompoundProcessor(factory.create(Map.of(), null, null, outerConfig, null)) ); Map innerConfig = new HashMap<>(); innerConfig.put("name", outerPipelineId); @@ -117,13 +117,14 @@ public void testThrowsOnRecursivePipelineInvocations() throws Exception { null, null, null, - new CompoundProcessor(factory.create(Map.of(), null, null, innerConfig)) + new CompoundProcessor(factory.create(Map.of(), null, null, innerConfig, null)) ); when(ingestService.getPipeline(outerPipelineId)).thenReturn(outer); when(ingestService.getPipeline(innerPipelineId)).thenReturn(inner); outerConfig.put("name", innerPipelineId); ElasticsearchException[] e = new ElasticsearchException[1]; - factory.create(Map.of(), null, null, outerConfig).execute(testIngestDocument, (result, e1) -> e[0] = (ElasticsearchException) e1); + factory.create(Map.of(), null, null, outerConfig, null) + .execute(testIngestDocument, (result, e1) -> e[0] = (ElasticsearchException) e1); assertEquals("Cycle detected for pipeline: inner", e[0].getRootCause().getMessage()); } @@ -136,7 +137,7 @@ public void testAllowsRepeatedPipelineInvocations() throws Exception { PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); Pipeline inner = new Pipeline(innerPipelineId, null, null, null, new CompoundProcessor()); when(ingestService.getPipeline(innerPipelineId)).thenReturn(inner); - Processor outerProc = factory.create(Map.of(), null, null, outerConfig); + Processor outerProc = factory.create(Map.of(), null, null, outerConfig, null); outerProc.execute(testIngestDocument, (result, e) -> {}); outerProc.execute(testIngestDocument, (result, e) -> {}); } @@ -150,11 +151,11 @@ public void testPipelineProcessorWithPipelineChain() throws Exception { Map pipeline1ProcessorConfig = new HashMap<>(); pipeline1ProcessorConfig.put("name", pipeline2Id); - PipelineProcessor pipeline1Processor = factory.create(Map.of(), null, null, pipeline1ProcessorConfig); + PipelineProcessor pipeline1Processor = factory.create(Map.of(), null, null, pipeline1ProcessorConfig, null); Map pipeline2ProcessorConfig = new HashMap<>(); pipeline2ProcessorConfig.put("name", pipeline3Id); - PipelineProcessor pipeline2Processor = factory.create(Map.of(), null, null, pipeline2ProcessorConfig); + PipelineProcessor pipeline2Processor = factory.create(Map.of(), null, null, pipeline2ProcessorConfig, null); LongSupplier relativeTimeProvider = mock(LongSupplier.class); when(relativeTimeProvider.getAsLong()).thenReturn(0L); diff --git a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java index 83fcc0d91cbb3..e9ec1f2d6260c 100644 --- a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.bulk.FailureStoreMetrics; import org.elasticsearch.action.bulk.SimulateBulkRequest; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.project.TestProjectResolvers; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; @@ -48,21 +49,21 @@ public void testGetPipeline() { Map processors = new HashMap<>(); processors.put( "processor1", - (factories, tag, description, config) -> new FakeProcessor("processor1", tag, description, ingestDocument -> {}) { + (factories, tag, description, config, projectId) -> new FakeProcessor("processor1", tag, description, ingestDocument -> {}) { } ); processors.put( "processor2", - (factories, tag, description, config) -> new FakeProcessor("processor2", tag, description, ingestDocument -> {}) { + (factories, tag, description, config, projectId) -> new FakeProcessor("processor2", tag, description, ingestDocument -> {}) { } ); processors.put( "processor3", - (factories, tag, description, config) -> new FakeProcessor("processor3", tag, description, ingestDocument -> {}) { + (factories, tag, description, config, projectId) -> new FakeProcessor("processor3", tag, description, ingestDocument -> {}) { } ); - IngestService ingestService = createWithProcessors(processors); final var projectId = randomProjectIdOrDefault(); + IngestService ingestService = createWithProcessors(projectId, processors); ingestService.innerUpdatePipelines(projectId, ingestMetadata); { // First we make sure that if there are no substitutions that we get our original pipeline back: @@ -113,7 +114,7 @@ public void testGetPipeline() { } } - private static IngestService createWithProcessors(Map processors) { + private static IngestService createWithProcessors(ProjectId projectId, Map processors) { Client client = mock(Client.class); ThreadPool threadPool = mock(ThreadPool.class); when(threadPool.generic()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); @@ -134,7 +135,7 @@ public Map getProcessors(final Processor.Parameters p client, null, FailureStoreMetrics.NOOP, - TestProjectResolvers.singleProjectOnly() + TestProjectResolvers.singleProject(projectId) ); } } diff --git a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java index 06e8739ee4a85..c218ceef88f89 100644 --- a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java @@ -306,7 +306,7 @@ public void testActualPipelineProcessor() throws Exception { }))); when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline); - PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig); + PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig, null); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -375,7 +375,7 @@ public void testActualPipelineProcessorWithTrueConditional() throws Exception { null, new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()), scriptService, - factory.create(Map.of(), "pipeline1", null, pipelineConfig2) + factory.create(Map.of(), "pipeline1", null, pipelineConfig2, null) ), new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key3, randomInt()); @@ -389,7 +389,7 @@ public void testActualPipelineProcessorWithTrueConditional() throws Exception { when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1); when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2); - PipelineProcessor pipelineProcessor = factory.create(Map.of(), "pipeline0", null, pipelineConfig0); + PipelineProcessor pipelineProcessor = factory.create(Map.of(), "pipeline0", null, pipelineConfig0, null); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -468,7 +468,7 @@ public void testActualPipelineProcessorWithFalseConditional() throws Exception { null, new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()), scriptService, - factory.create(Map.of(), null, null, pipelineConfig2) + factory.create(Map.of(), null, null, pipelineConfig2, null) ), new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key3, randomInt()); @@ -482,7 +482,7 @@ public void testActualPipelineProcessorWithFalseConditional() throws Exception { when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1); when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2); - PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0); + PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0, null); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -545,7 +545,7 @@ public void testActualPipelineProcessorWithHandledFailure() throws Exception { )); when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline); - PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig); + PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig, null); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -608,7 +608,7 @@ public void testActualPipelineProcessorWithUnhandledFailure() throws Exception { ); when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline); - PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig); + PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig, null); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -650,7 +650,7 @@ public void testActualPipelineProcessorWithCycle() throws Exception { null, null, null, - new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig2)) + new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig2, null)) ); Pipeline pipeline2 = new Pipeline( @@ -658,13 +658,13 @@ public void testActualPipelineProcessorWithCycle() throws Exception { null, null, null, - new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig1)) + new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig1, null)) ); when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1); when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2); - PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0); + PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0, null); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -694,7 +694,7 @@ public void testActualPipelineProcessorNested() throws Exception { null, null, null, - new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig)) + new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig, null)) ); when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline); } @@ -708,7 +708,7 @@ public void testActualPipelineProcessorNested() throws Exception { String firstPipelineId = "pipeline0"; Map firstPipelineConfig = new HashMap<>(); firstPipelineConfig.put("name", firstPipelineId); - PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig); + PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig, null); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -747,7 +747,7 @@ public void testActualPipelineProcessorNestedTooManyPipelines() throws Exception null, null, null, - new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig)) + new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig, null)) ); when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline); } @@ -761,7 +761,7 @@ public void testActualPipelineProcessorNestedTooManyPipelines() throws Exception String firstPipelineId = "pipeline0"; Map firstPipelineConfig = new HashMap<>(); firstPipelineConfig.put("name", firstPipelineId); - PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig); + PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig, null); CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); @@ -789,7 +789,7 @@ public void testActualPipelineProcessorRepeatedInvocation() throws Exception { PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); String key1 = randomAlphaOfLength(10); - PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig); + PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig, null); Pipeline pipeline = new Pipeline(pipelineId, null, null, null, new CompoundProcessor(new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key1, randomInt()); }))); diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java index faabf22070820..e3469caeefde2 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java @@ -20,7 +20,7 @@ public class IngestTestPlugin extends Plugin implements IngestPlugin { @Override public Map getProcessors(Processor.Parameters parameters) { - return Map.of("test", (factories, tag, description, config) -> new TestProcessor("id", "test", "description", doc -> { + return Map.of("test", (factories, tag, description, config, projectId) -> new TestProcessor("id", "test", "description", doc -> { doc.setFieldValue("processed", true); if (doc.hasField("fail") && doc.getFieldValue("fail", Boolean.class)) { throw new IllegalArgumentException("test processor failed"); diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java index b38dc7a0be95a..aa1ed922479ea 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java @@ -9,6 +9,8 @@ package org.elasticsearch.ingest; +import org.elasticsearch.cluster.metadata.ProjectId; + import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; @@ -102,7 +104,8 @@ public TestProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { return new TestProcessor(processorTag, "test-processor", description, ingestDocument -> {}); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidatorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidatorTests.java index 04e04f65b1182..7a318af6d6e7b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidatorTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidatorTests.java @@ -599,7 +599,8 @@ public void testCheck_GivenMissingDestPipeline() throws Exception { Arrays.asList(Collections.singletonMap("test", processorConfig0), Collections.singletonMap("test", processorConfig1)) ); Map processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory()); - Pipeline pipeline = Pipeline.create("missing-pipeline", pipelineConfig, processorRegistry, null); + var projectId = randomProjectIdOrDefault(); + Pipeline pipeline = Pipeline.create("missing-pipeline", pipelineConfig, processorRegistry, null, projectId); when(ingestService.getPipeline("missing-pipeline")).thenReturn(pipeline); assertValidation( diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java index d11ca41b3fbaa..170571269316f 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.xpack.core.enrich.action.EnrichStatsAction; @@ -89,6 +90,7 @@ private Cache createCache(long maxWeight, ToLongBiFunction * @param searchResponseFetcher The function used to compute the value to be put in the cache, if there is no value in the cache already * @param listener A listener to be notified of the value in the cache */ + @FixForMultiProject(description = "The enrich cache will currently leak data between projects. We need to either disable or fix it.") public void computeIfAbsent( String enrichIndex, Object lookupValue, diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java index 6aef15d2d75f2..f17ac40cec916 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; @@ -115,19 +116,20 @@ public void coordinatePolicyExecution( } public void runPolicyLocally( + ProjectId projectId, ExecuteEnrichPolicyTask task, String policyName, String enrichIndexName, ActionListener listener ) { try { - EnrichPolicy policy = EnrichStore.getPolicy(policyName, clusterService.state().metadata().getProject()); + EnrichPolicy policy = EnrichStore.getPolicy(policyName, clusterService.state().metadata().getProject(projectId)); if (policy == null) { throw new ResourceNotFoundException("policy [{}] does not exist", policyName); } task.setStatus(new ExecuteEnrichPolicyStatus(ExecuteEnrichPolicyStatus.PolicyPhases.SCHEDULED)); - var policyRunner = createPolicyRunner(policyName, policy, enrichIndexName, task); + var policyRunner = createPolicyRunner(projectId, policyName, policy, enrichIndexName, task); threadPool.executor(ThreadPool.Names.GENERIC) .execute(ActionRunnable.wrap(ActionListener.assertOnce(listener), policyRunner::run)); } catch (Exception e) { @@ -209,12 +211,14 @@ public void onFailure(Exception exception) { } private EnrichPolicyRunner createPolicyRunner( + ProjectId projectId, String policyName, EnrichPolicy policy, String enrichIndexName, ExecuteEnrichPolicyTask task ) { return new EnrichPolicyRunner( + projectId, policyName, policy, task, diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyReindexPipeline.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyReindexPipeline.java index ebc08049953bc..012c05afa44a1 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyReindexPipeline.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyReindexPipeline.java @@ -12,7 +12,7 @@ import org.elasticsearch.action.ingest.PutPipelineTransportAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.ingest.IngestMetadata; import org.elasticsearch.ingest.PipelineConfiguration; @@ -47,11 +47,11 @@ static String pipelineName() { /** * Checks if the current version of the pipeline definition is installed in the cluster - * @param clusterState The cluster state to check + * @param project The project metadata to check * @return true if a pipeline exists that is compatible with this version of Enrich, false otherwise */ - static boolean exists(ClusterState clusterState) { - final IngestMetadata ingestMetadata = clusterState.getMetadata().getProject().custom(IngestMetadata.TYPE); + static boolean exists(ProjectMetadata project) { + final IngestMetadata ingestMetadata = project.custom(IngestMetadata.TYPE); // we ensure that we both have the pipeline and its version represents the current (or later) version if (ingestMetadata != null) { final PipelineConfiguration pipeline = ingestMetadata.getPipelines().get(pipelineName()); diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java index 36e6914b844bc..c0811df8a0123 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java @@ -40,10 +40,11 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.FilterClient; import org.elasticsearch.client.internal.OriginSettingClient; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; @@ -102,6 +103,7 @@ public class EnrichPolicyRunner { */ static final TimeValue ENRICH_MASTER_REQUEST_TIMEOUT = TimeValue.THIRTY_SECONDS; + private final ProjectId projectId; private final String policyName; private final EnrichPolicy policy; private final ExecuteEnrichPolicyTask task; @@ -114,6 +116,7 @@ public class EnrichPolicyRunner { private final int maxForceMergeAttempts; EnrichPolicyRunner( + ProjectId projectId, String policyName, EnrichPolicy policy, ExecuteEnrichPolicyTask task, @@ -125,6 +128,7 @@ public class EnrichPolicyRunner { int fetchSize, int maxForceMergeAttempts ) { + this.projectId = projectId; this.policyName = Objects.requireNonNull(policyName); this.policy = Objects.requireNonNull(policy); this.task = Objects.requireNonNull(task); @@ -480,7 +484,7 @@ private void prepareAndCreateEnrichIndex( private void prepareReindexOperation(ActionListener listener) { // Check to make sure that the enrich pipeline exists, and create it if it is missing. - if (EnrichPolicyReindexPipeline.exists(clusterService.state())) { + if (EnrichPolicyReindexPipeline.exists(clusterService.state().getMetadata().getProject(projectId))) { listener.onResponse(null); } else { EnrichPolicyReindexPipeline.create(enrichOriginClient(), listener); @@ -700,8 +704,8 @@ private void waitForIndexGreen(ActionListener listener) { * and recreated with invalid mappings/data. We validate that the mapping exists and that it contains the expected meta fields on it to * guard against accidental removal and recreation during policy execution. */ - private void validateIndexBeforePromotion(String destinationIndexName, ClusterState clusterState) { - IndexMetadata destinationIndex = clusterState.metadata().getProject().index(destinationIndexName); + private void validateIndexBeforePromotion(String destinationIndexName, ProjectMetadata project) { + IndexMetadata destinationIndex = project.index(destinationIndexName); if (destinationIndex == null) { throw new IndexNotFoundException( "was not able to promote it as part of executing enrich policy [" + policyName + "]", @@ -749,12 +753,12 @@ private void updateEnrichPolicyAlias(ActionListener list String enrichIndexBase = EnrichPolicy.getBaseName(policyName); logger.debug("Policy [{}]: Promoting new enrich index [{}] to alias [{}]", policyName, enrichIndexName, enrichIndexBase); GetAliasesRequest aliasRequest = new GetAliasesRequest(ENRICH_MASTER_REQUEST_TIMEOUT, enrichIndexBase); - ClusterState clusterState = clusterService.state(); - validateIndexBeforePromotion(enrichIndexName, clusterState); - String[] concreteIndices = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(clusterState, aliasRequest); + final var project = clusterService.state().metadata().getProject(projectId); + validateIndexBeforePromotion(enrichIndexName, project); + String[] concreteIndices = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(project, aliasRequest); String[] aliases = aliasRequest.aliases(); IndicesAliasesRequest aliasToggleRequest = new IndicesAliasesRequest(ENRICH_MASTER_REQUEST_TIMEOUT, ENRICH_MASTER_REQUEST_TIMEOUT); - String[] indices = clusterState.metadata().getProject().findAliases(aliases, concreteIndices).keySet().toArray(new String[0]); + String[] indices = project.findAliases(aliases, concreteIndices).keySet().toArray(new String[0]); if (indices.length > 0) { aliasToggleRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove().indices(indices).alias(enrichIndexBase)); } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java index 97c53d62d3a2d..21829e2a884a3 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java @@ -14,6 +14,8 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.geo.Orientation; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.xcontent.support.XContentMapValues; @@ -50,20 +52,26 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer processorFactories, String tag, String description, Map config) - throws Exception { + public Processor create( + Map processorFactories, + String tag, + String description, + Map config, + ProjectId projectId + ) throws Exception { final String policyName = ConfigurationUtils.readStringProperty(TYPE, tag, config, "policy_name"); final String indexAlias = EnrichPolicy.getBaseName(policyName); if (metadata == null) { throw new IllegalStateException("enrich processor factory has not yet been initialized with cluster state"); } - IndexAbstraction indexAbstraction = metadata.getProject().getIndicesLookup().get(indexAlias); + final var project = metadata.getProject(projectId); + IndexAbstraction indexAbstraction = project.getIndicesLookup().get(indexAlias); if (indexAbstraction == null) { throw new IllegalArgumentException("no enrich index exists for policy with name [" + policyName + "]"); } assert indexAbstraction.getType() == IndexAbstraction.Type.ALIAS; assert indexAbstraction.getIndices().size() == 1; - IndexMetadata imd = metadata.getProject().index(indexAbstraction.getIndices().get(0)); + IndexMetadata imd = project.index(indexAbstraction.getIndices().get(0)); Map mappingAsMap = imd.mapping().sourceAsMap(); String policyType = (String) XContentMapValues.extractValue( @@ -80,7 +88,7 @@ public Processor create(Map processorFactories, Strin if (maxMatches < 1 || maxMatches > 128) { throw ConfigurationUtils.newConfigurationException(TYPE, tag, "max_matches", "should be between 1 and 128"); } - var searchRunner = createSearchRunner(indexAlias, client, enrichCache); + var searchRunner = createSearchRunner(project, indexAlias); switch (policyType) { case EnrichPolicy.MATCH_TYPE: case EnrichPolicy.RANGE_TYPE: @@ -125,12 +133,12 @@ public void accept(ClusterState state) { metadata = state.getMetadata(); } - private SearchRunner createSearchRunner(String indexAlias, Client client, EnrichCache enrichCache) { + private SearchRunner createSearchRunner(ProjectMetadata project, String indexAlias) { Client originClient = new OriginSettingClient(client, ENRICH_ORIGIN); return (value, maxMatches, reqSupplier, handler) -> { // intentionally non-locking for simplicity...it's OK if we re-put the same key/value in the cache during a race condition. enrichCache.computeIfAbsent( - getEnrichIndexKey(indexAlias), + getEnrichIndexKey(project, indexAlias), value, maxMatches, (searchResponseActionListener) -> originClient.execute( @@ -143,8 +151,8 @@ private SearchRunner createSearchRunner(String indexAlias, Client client, Enrich }; } - private String getEnrichIndexKey(String indexAlias) { - IndexAbstraction ia = metadata.getProject().getIndicesLookup().get(indexAlias); + private String getEnrichIndexKey(ProjectMetadata project, String indexAlias) { + IndexAbstraction ia = project.getIndicesLookup().get(indexAlias); if (ia == null) { throw new IndexNotFoundException("no generated enrich index [" + indexAlias + "]"); } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyAction.java index fd07dce89d370..3149a7f0cac03 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.io.stream.StreamInput; @@ -108,6 +109,7 @@ public static class Transport extends HandledTransportAction private final ClusterService clusterService; private final TransportService transportService; + private final ProjectResolver projectResolver; private final EnrichPolicyExecutor policyExecutor; private final AtomicInteger nodeGenerator = new AtomicInteger(Randomness.get().nextInt()); @@ -116,11 +118,13 @@ public Transport( TransportService transportService, ActionFilters actionFilters, ClusterService clusterService, + ProjectResolver projectResolver, EnrichPolicyExecutor policyExecutor ) { super(NAME, transportService, actionFilters, Request::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.clusterService = clusterService; this.transportService = transportService; + this.projectResolver = projectResolver; this.policyExecutor = policyExecutor; } @@ -182,13 +186,19 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, } ); } - policyExecutor.runPolicyLocally(task, request.getName(), request.getEnrichIndexName(), ActionListener.wrap(result -> { - taskManager.unregister(task); - listener.onResponse(result); - }, e -> { - taskManager.unregister(task); - listener.onFailure(e); - })); + policyExecutor.runPolicyLocally( + projectResolver.getProjectId(), + task, + request.getName(), + request.getEnrichIndexName(), + ActionListener.wrap(result -> { + taskManager.unregister(task); + listener.onResponse(result); + }, e -> { + taskManager.unregister(task); + listener.onFailure(e); + }) + ); if (request.isWaitForCompletion() == false) { TaskId taskId = new TaskId(clusterState.nodes().getLocalNodeId(), task.getId()); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutorTests.java index 06f9eb21fe2dc..b9bd3689b36df 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutorTests.java @@ -20,7 +20,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; @@ -385,8 +385,11 @@ protected void public void testRunPolicyLocallyMissingPolicy() { EnrichPolicy enrichPolicy = EnrichPolicyTests.randomEnrichPolicy(XContentType.JSON); + final var projectId = randomProjectIdOrDefault(); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) - .metadata(Metadata.builder().putCustom(EnrichMetadata.TYPE, new EnrichMetadata(Map.of("id", enrichPolicy))).build()) + .putProjectMetadata( + ProjectMetadata.builder(projectId).putCustom(EnrichMetadata.TYPE, new EnrichMetadata(Map.of("id", enrichPolicy))).build() + ) .build(); ClusterService clusterService = mock(ClusterService.class); when(clusterService.state()).thenReturn(clusterState); @@ -405,7 +408,7 @@ public void testRunPolicyLocallyMissingPolicy() { ExecuteEnrichPolicyTask task = mock(ExecuteEnrichPolicyTask.class); Exception e = expectThrows( ResourceNotFoundException.class, - () -> testExecutor.runPolicyLocally(task, "my-policy", ".enrich-my-policy-123456789", null) + () -> testExecutor.runPolicyLocally(projectId, task, "my-policy", ".enrich-my-policy-123456789", null) ); assertThat(e.getMessage(), equalTo("policy [my-policy] does not exist")); } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java index 3a27d1a3fcb1a..2dfe10b3fbcf8 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.FilterClient; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; @@ -1787,6 +1788,7 @@ public String getDescription() { .field("field5", "value5") .endObject(); EnrichPolicyRunner enrichPolicyRunner = new EnrichPolicyRunner( + Metadata.DEFAULT_PROJECT_ID, policyName, policy, task, @@ -1967,6 +1969,7 @@ protected void }; EnrichPolicyRunner enrichPolicyRunner = new EnrichPolicyRunner( + Metadata.DEFAULT_PROJECT_ID, policyName, policy, task, @@ -2088,6 +2091,7 @@ protected void }; EnrichPolicyRunner enrichPolicyRunner = new EnrichPolicyRunner( + Metadata.DEFAULT_PROJECT_ID, policyName, policy, task, @@ -2490,6 +2494,7 @@ public String getDescription() { }); ExecuteEnrichPolicyTask task = ((ExecuteEnrichPolicyTask) asyncTask); return new EnrichPolicyRunner( + Metadata.DEFAULT_PROJECT_ID, policyName, policy, task, diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java index 67c88a024e740..c6c91b8997a67 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java @@ -16,6 +16,8 @@ import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.core.Strings; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexVersion; @@ -46,10 +48,12 @@ public class EnrichProcessorFactoryTests extends ESTestCase { private ScriptService scriptService; private EnrichCache enrichCache = new EnrichCache(0L); + private ProjectId projectId; @Before public void initializeScriptService() { scriptService = mock(ScriptService.class); + projectId = randomProjectIdOrDefault(); } public void testCreateProcessorInstance() throws Exception { @@ -86,7 +90,7 @@ public void testCreateProcessorInstance() throws Exception { randomValues.add(new Tuple<>(randomFrom(enrichValues), randomAlphaOfLength(4))); } - MatchProcessor result = (MatchProcessor) factory.create(Collections.emptyMap(), "_tag", null, config); + MatchProcessor result = (MatchProcessor) factory.create(Collections.emptyMap(), "_tag", null, config, projectId); assertThat(result, notNullValue()); assertThat(result.getPolicyName(), equalTo("majestic")); assertThat(result.getField(), equalTo("host")); @@ -109,7 +113,7 @@ public void testCreateProcessorInstance() throws Exception { public void testPolicyDoesNotExist() { List enrichValues = List.of("globalRank", "tldRank", "tld"); EnrichProcessorFactory factory = new EnrichProcessorFactory(null, scriptService, enrichCache); - factory.metadata = Metadata.builder().build(); + factory.metadata = Metadata.builder().put(ProjectMetadata.builder(projectId)).build(); Map config = new HashMap<>(); config.put("policy_name", "majestic"); @@ -131,7 +135,10 @@ public void testPolicyDoesNotExist() { } config.put("set_from", valuesConfig); - Exception e = expectThrows(IllegalArgumentException.class, () -> factory.create(Collections.emptyMap(), "_tag", null, config)); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> factory.create(Collections.emptyMap(), "_tag", null, config, projectId) + ); assertThat(e.getMessage(), equalTo("no enrich index exists for policy with name [majestic]")); } @@ -158,7 +165,10 @@ public void testPolicyNameMissing() { } config.put("set_from", valuesConfig); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(Collections.emptyMap(), "_tag", null, config)); + Exception e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(Collections.emptyMap(), "_tag", null, config, projectId) + ); assertThat(e.getMessage(), equalTo("[policy_name] required property is missing")); } @@ -179,7 +189,10 @@ public void testUnsupportedPolicy() throws Exception { config.put("ignore_missing", keyIgnoreMissing); } - Exception e = expectThrows(IllegalArgumentException.class, () -> factory.create(Collections.emptyMap(), "_tag", null, config)); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> factory.create(Collections.emptyMap(), "_tag", null, config, projectId) + ); assertThat(e.getMessage(), equalTo("unsupported policy type [unsupported]")); } } @@ -197,7 +210,7 @@ public void testCompactEnrichValuesFormat() throws Exception { config.put("field", "host"); config.put("target_field", "entry"); - MatchProcessor result = (MatchProcessor) factory.create(Collections.emptyMap(), "_tag", null, config); + MatchProcessor result = (MatchProcessor) factory.create(Collections.emptyMap(), "_tag", null, config, projectId); assertThat(result, notNullValue()); assertThat(result.getPolicyName(), equalTo("majestic")); assertThat(result.getField(), equalTo("host")); @@ -215,7 +228,10 @@ public void testNoTargetField() throws Exception { config1.put("policy_name", "majestic"); config1.put("field", "host"); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(Collections.emptyMap(), "_tag", null, config1)); + Exception e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(Collections.emptyMap(), "_tag", null, config1, projectId) + ); assertThat(e.getMessage(), equalTo("[target_field] required property is missing")); } @@ -231,7 +247,10 @@ public void testIllegalMaxMatches() throws Exception { config.put("target_field", "entry"); config.put("max_matches", randomBoolean() ? between(-2048, 0) : between(129, 2048)); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(Collections.emptyMap(), "_tag", null, config)); + Exception e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(Collections.emptyMap(), "_tag", null, config, projectId) + ); assertThat(e.getMessage(), equalTo("[max_matches] should be between 1 and 128")); } @@ -272,7 +291,7 @@ protected void VersionType.INTERNAL, Map.of("domain", "elastic.co") ); - MatchProcessor processor = (MatchProcessor) factory.create(Collections.emptyMap(), "_tag", null, config); + MatchProcessor processor = (MatchProcessor) factory.create(Collections.emptyMap(), "_tag", null, config, projectId); // A search is performed and that is cached: IngestDocument[] result = new IngestDocument[1]; @@ -306,14 +325,14 @@ protected void } } - static Metadata createMetadata(String name, EnrichPolicy policy) { + private Metadata createMetadata(String name, EnrichPolicy policy) { IndexMetadata.Builder builder = IndexMetadata.builder(EnrichPolicy.getBaseName(name) + "-1"); builder.settings(indexSettings(IndexVersion.current(), 1, 0)); builder.putMapping(Strings.format(""" {"_meta": {"enrich_match_field": "%s", "enrich_policy_type": "%s"}} """, policy.getMatchField(), policy.getType())); builder.putAlias(AliasMetadata.builder(EnrichPolicy.getBaseName(name)).build()); - return Metadata.builder().put(builder).build(); + return Metadata.builder().put(ProjectMetadata.builder(projectId).put(builder)).build(); } } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java index 76577b12cd8d9..99df6c7c1c87d 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.project.TestProjectResolvers; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; @@ -30,7 +31,13 @@ public class InternalExecutePolicyActionTests extends ESTestCase { @Before public void instantiateTransportAction() { TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - transportAction = new InternalExecutePolicyAction.Transport(transportService, mock(ActionFilters.class), null, null); + transportAction = new InternalExecutePolicyAction.Transport( + transportService, + mock(ActionFilters.class), + null, + TestProjectResolvers.singleProjectOnly(), + null + ); } public void testSelectNodeForPolicyExecution() { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java index e50d67e068756..2200de74c310f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -415,7 +416,8 @@ public InferenceProcessor create( Map processorFactories, String tag, String description, - Map config + Map config, + ProjectId projectId ) { final var currentInferenceProcessors = InferenceProcessorInfoExtractor.countInferenceProcessors(clusterState); if (this.maxIngestProcessors <= currentInferenceProcessors) { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java index 8b909248f3002..28d39d3f1682c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.bulk.FailureStoreMetrics; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.project.TestProjectResolvers; import org.elasticsearch.cluster.routing.OperationRouting; @@ -83,7 +84,8 @@ public Processor create( Map processorFactories, String tag, String description, - Map config + Map config, + ProjectId projectId ) { return new NotInferenceProcessor(); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java index 7ffddc9721bdf..4b0c4a5f7e814 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java @@ -126,7 +126,7 @@ public void testCreateProcessorWithTooManyExisting() { ElasticsearchStatusException ex = expectThrows( ElasticsearchStatusException.class, - () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, Collections.emptyMap()) + () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, Collections.emptyMap(), null) ); assertThat( @@ -157,7 +157,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() { ElasticsearchStatusException ex = expectThrows( ElasticsearchStatusException.class, - () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config) + () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config, null) ); assertThat( ex.getMessage(), @@ -179,7 +179,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() { }; ex = expectThrows( ElasticsearchStatusException.class, - () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config2) + () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config2, null) ); assertThat(ex.getMessage(), equalTo("inference_config must be an object with one inference type mapped to an object.")); @@ -193,7 +193,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() { }; ex = expectThrows( ElasticsearchStatusException.class, - () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config3) + () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config3, null) ); assertThat(ex.getMessage(), equalTo("inference_config must be an object with one inference type mapped to an object.")); } @@ -224,7 +224,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() { ElasticsearchException ex = expectThrows( ElasticsearchException.class, - () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, regression) + () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, regression, null) ); assertThat( ex.getMessage(), @@ -248,7 +248,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() { ex = expectThrows( ElasticsearchException.class, - () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, classification) + () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, classification, null) ); assertThat( ex.getMessage(), @@ -322,7 +322,7 @@ public void testCreateProcessor() { } }; - var processor = processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, regression); + var processor = processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, regression, null); assertFalse(processor.isConfiguredWithInputsFields()); assertEquals("my_model", processor.getModelId()); assertEquals("result", processor.getTargetField()); @@ -344,7 +344,7 @@ public void testCreateProcessor() { } }; - processor = processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, classification); + processor = processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, classification, null); assertFalse(processor.isConfiguredWithInputsFields()); Map mininmal = new HashMap<>() { @@ -354,7 +354,7 @@ public void testCreateProcessor() { } }; - processor = processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, mininmal); + processor = processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, mininmal, null); assertFalse(processor.isConfiguredWithInputsFields()); assertEquals("my_model", processor.getModelId()); assertEquals("result", processor.getTargetField()); @@ -381,7 +381,7 @@ public void testCreateProcessorWithFieldMap() { } }; - var processor = processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config); + var processor = processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config, null); assertFalse(processor.isConfiguredWithInputsFields()); assertEquals("my_model", processor.getModelId()); assertEquals("result", processor.getTargetField()); @@ -414,7 +414,7 @@ public void testCreateProcessorWithInputOutputs() { inputOutputs.add(input2); config.put(InferenceProcessor.INPUT_OUTPUT, inputOutputs); - var processor = processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config); + var processor = processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, config, null); assertTrue(processor.isConfiguredWithInputsFields()); assertEquals("my_model", processor.getModelId()); var configuredInputs = processor.getInputs(); @@ -451,7 +451,7 @@ public void testCreateProcessorWithDuplicateFields() { Exception ex = expectThrows( Exception.class, - () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, regression) + () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, regression, null) ); assertThat(ex.getMessage(), equalTo("Invalid inference config. " + "More than one field is configured as [warning]")); } @@ -481,7 +481,7 @@ public void testCreateProcessorWithIgnoreMissing() { Exception ex = expectThrows( Exception.class, - () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, regression) + () -> processorFactory.create(Collections.emptyMap(), "my_inference_processor", null, regression, null) ); assertThat(ex.getMessage(), equalTo("Invalid inference config. " + "More than one field is configured as [warning]")); } @@ -538,7 +538,7 @@ public void testCreateProcessorWithIncompatibleTargetFieldSetting() { ElasticsearchParseException ex = expectThrows( ElasticsearchParseException.class, - () -> processorFactory.create(Collections.emptyMap(), "processor_with_inputs", null, config) + () -> processorFactory.create(Collections.emptyMap(), "processor_with_inputs", null, config, null) ); assertThat( ex.getMessage(), @@ -580,7 +580,7 @@ public void testCreateProcessorWithIncompatibleResultFieldSetting() { ElasticsearchParseException ex = expectThrows( ElasticsearchParseException.class, - () -> processorFactory.create(Collections.emptyMap(), "processor_with_inputs", null, config) + () -> processorFactory.create(Collections.emptyMap(), "processor_with_inputs", null, config, null) ); assertThat( ex.getMessage(), @@ -645,7 +645,7 @@ public void testCreateProcessorWithInputFields() { config.put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(inferenceConfigType, Collections.emptyMap())); } - var inferenceProcessor = processorFactory.create(Collections.emptyMap(), "processor_with_inputs", null, config); + var inferenceProcessor = processorFactory.create(Collections.emptyMap(), "processor_with_inputs", null, config, null); assertEquals("my_model", inferenceProcessor.getModelId()); assertTrue(inferenceProcessor.isConfiguredWithInputsFields()); @@ -689,7 +689,7 @@ public void testCreateProcessorWithInputFieldSingleOrList() { ); } - var inferenceProcessor = processorFactory.create(Collections.emptyMap(), "processor_with_single_input", null, config); + var inferenceProcessor = processorFactory.create(Collections.emptyMap(), "processor_with_single_input", null, config, null); assertEquals("my_model", inferenceProcessor.getModelId()); assertTrue(inferenceProcessor.isConfiguredWithInputsFields()); @@ -717,7 +717,7 @@ public void testCreateProcessorWithInputFieldWrongType() { var e = expectThrows( ElasticsearchParseException.class, - () -> processorFactory.create(Collections.emptyMap(), "processor_with_bad_config", null, config) + () -> processorFactory.create(Collections.emptyMap(), "processor_with_bad_config", null, config, null) ); assertThat(e.getMessage(), containsString("[input_output] property isn't a list of maps")); } @@ -728,7 +728,7 @@ public void testCreateProcessorWithInputFieldWrongType() { var e = expectThrows( ElasticsearchParseException.class, - () -> processorFactory.create(Collections.emptyMap(), "processor_with_bad_config", null, config) + () -> processorFactory.create(Collections.emptyMap(), "processor_with_bad_config", null, config, null) ); assertThat(e.getMessage(), containsString("[input_output] property isn't a map or list of maps")); } @@ -741,7 +741,7 @@ public void testCreateProcessorWithInputFieldWrongType() { var e = expectThrows( ElasticsearchParseException.class, - () -> processorFactory.create(Collections.emptyMap(), "processor_with_bad_config", null, config) + () -> processorFactory.create(Collections.emptyMap(), "processor_with_bad_config", null, config, null) ); assertThat(e.getMessage(), containsString("[input_field] required property is missing")); } @@ -753,7 +753,7 @@ public void testCreateProcessorWithInputFieldWrongType() { var e = expectThrows( ElasticsearchParseException.class, - () -> processorFactory.create(Collections.emptyMap(), "processor_with_bad_config", null, config) + () -> processorFactory.create(Collections.emptyMap(), "processor_with_bad_config", null, config, null) ); assertThat(e.getMessage(), containsString("[input_output] property cannot be empty at least one is required")); } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java index e4b030dba90f3..9316197590e9e 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.monitoring.test; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; @@ -51,7 +52,8 @@ public Processor create( Map processorFactories, String tag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { // read fields so the processor succeeds for (final String field : fields) { diff --git a/x-pack/plugin/redact/src/main/java/org/elasticsearch/xpack/redact/RedactProcessor.java b/x-pack/plugin/redact/src/main/java/org/elasticsearch/xpack/redact/RedactProcessor.java index c378b822ce0b0..6884725d2f84b 100644 --- a/x-pack/plugin/redact/src/main/java/org/elasticsearch/xpack/redact/RedactProcessor.java +++ b/x-pack/plugin/redact/src/main/java/org/elasticsearch/xpack/redact/RedactProcessor.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.grok.Grok; import org.elasticsearch.grok.GrokBuiltinPatterns; import org.elasticsearch.grok.GrokCaptureExtracter; @@ -410,7 +411,8 @@ public RedactProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); List matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns"); diff --git a/x-pack/plugin/redact/src/test/java/org/elasticsearch/xpack/redact/RedactProcessorFactoryTests.java b/x-pack/plugin/redact/src/test/java/org/elasticsearch/xpack/redact/RedactProcessorFactoryTests.java index affcc72614aa8..e5423c763ebe3 100644 --- a/x-pack/plugin/redact/src/test/java/org/elasticsearch/xpack/redact/RedactProcessorFactoryTests.java +++ b/x-pack/plugin/redact/src/test/java/org/elasticsearch/xpack/redact/RedactProcessorFactoryTests.java @@ -44,7 +44,7 @@ public void testPatternNotSet() { Map config = new HashMap<>(); config.put("field", "_field"); config.put("patterns", List.of()); - ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, null, null, config)); + ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, null, null, config, null)); assertThat(e.getMessage(), containsString("List of patterns must not be empty")); } @@ -55,7 +55,7 @@ public void testCreateWithCustomPatterns() throws Exception { config.put("field", "_field"); config.put("patterns", List.of("%{MY_PATTERN:name}!")); config.put("pattern_definitions", Map.of("MY_PATTERN", "foo")); - RedactProcessor processor = factory.create(null, null, null, config); + RedactProcessor processor = factory.create(null, null, null, config, null); assertThat(processor.getGroks(), not(empty())); assertThat(processor.getGroks().get(0).match("foo!"), equalTo(true)); } @@ -71,7 +71,7 @@ public void testConfigKeysRemoved() throws Exception { config.put("trace_redact", true); config.put("extra", "unused"); - factory.create(null, null, null, config); + factory.create(null, null, null, config, null); assertThat(config.entrySet(), hasSize(1)); assertEquals("unused", config.get("extra")); } @@ -87,7 +87,7 @@ public void testSkipIfUnlicensed() throws Exception { // since skip_if_unlicensed is true, we can use the redact processor regardless of the license state XPackLicenseState licenseState = randomBoolean() ? mockLicenseState() : mockNotAllowedLicenseState(); RedactProcessor.Factory factory = new RedactProcessor.Factory(licenseState, MatcherWatchdog.noop()); - RedactProcessor processor = factory.create(null, null, null, config); + RedactProcessor processor = factory.create(null, null, null, config, null); processor.extraValidation(); assertThat(processor.getSkipIfUnlicensed(), equalTo(true)); } @@ -103,7 +103,7 @@ public void testSkipIfUnlicensed() throws Exception { // regardless of default/explicit, the license must be sufficient for the feature RedactProcessor.Factory factory = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()); - RedactProcessor processor = factory.create(null, null, null, config); + RedactProcessor processor = factory.create(null, null, null, config, null); processor.extraValidation(); assertThat(processor.getSkipIfUnlicensed(), equalTo(false)); } @@ -118,7 +118,7 @@ public void testSkipIfUnlicensed() throws Exception { } // if skip_if_unlicensed is false, then the license must allow for redact to be used in order to pass the extra validation RedactProcessor.Factory factory = new RedactProcessor.Factory(mockNotAllowedLicenseState(), MatcherWatchdog.noop()); - RedactProcessor processor = factory.create(null, null, null, config); + RedactProcessor processor = factory.create(null, null, null, config, null); ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> processor.extraValidation()); assertThat(e.getMessage(), containsString("[skip_if_unlicensed] current license is non-compliant for [redact_processor]")); } diff --git a/x-pack/plugin/redact/src/test/java/org/elasticsearch/xpack/redact/RedactProcessorTests.java b/x-pack/plugin/redact/src/test/java/org/elasticsearch/xpack/redact/RedactProcessorTests.java index bf287735d9fc3..d9a5c6ba5cad0 100644 --- a/x-pack/plugin/redact/src/test/java/org/elasticsearch/xpack/redact/RedactProcessorTests.java +++ b/x-pack/plugin/redact/src/test/java/org/elasticsearch/xpack/redact/RedactProcessorTests.java @@ -47,7 +47,7 @@ public void testMatchRedact() throws Exception { var config = new HashMap(); config.put("field", "to_redact"); config.put("patterns", List.of("%{EMAILADDRESS:EMAIL}")); - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config, null); var groks = processor.getGroks(); { @@ -71,7 +71,7 @@ public void testMatchRedact() throws Exception { config.put("field", "to_redact"); config.put("patterns", List.of("%{CREDIT_CARD:CREDIT_CARD}")); config.put("pattern_definitions", Map.of("CREDIT_CARD", "\\b(?:\\d[ -]*?){13,16}\\b")); - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config, null); var groks = processor.getGroks(); { @@ -101,7 +101,7 @@ public void testMatchRedact() throws Exception { config.put("field", "to_redact"); config.put("patterns", List.of("%{CREDIT_CARD:CREDIT_CARD}")); config.put("pattern_definitions", Map.of("CREDIT_CARD", "\\d{4}[ -]\\d{4}[ -]\\d{4}[ -]\\d{4}")); - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config, null); var grok = processor.getGroks().get(0); String input = "1001-1002-1003-1004 2001-1002-1003-1004 3001-1002-1003-1004 4001-1002-1003-1004"; @@ -113,7 +113,7 @@ public void testMatchRedact() throws Exception { config.put("field", "to_redact"); config.put("patterns", List.of("%{NUMBER:NUMBER}")); config.put("pattern_definitions", Map.of("NUMBER", "\\d{4}")); - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config, null); var grok = processor.getGroks().get(0); String input = "1001"; @@ -127,7 +127,7 @@ public void testMatchRedactMultipleGroks() throws Exception { config.put("field", "to_redact"); config.put("patterns", List.of("%{EMAILADDRESS:EMAIL}", "%{IP:IP_ADDRESS}", "%{CREDIT_CARD:CREDIT_CARD}")); config.put("pattern_definitions", Map.of("CREDIT_CARD", "\\d{4}[ -]\\d{4}[ -]\\d{4}[ -]\\d{4}")); - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config, null); var groks = processor.getGroks(); { @@ -142,7 +142,7 @@ public void testRedact() throws Exception { config.put("field", "to_redact"); config.put("patterns", List.of("%{EMAILADDRESS:EMAIL}", "%{IP:IP_ADDRESS}", "%{CREDIT_CARD:CREDIT_CARD}")); config.put("pattern_definitions", Map.of("CREDIT_CARD", "\\d{4}[ -]\\d{4}[ -]\\d{4}[ -]\\d{4}")); - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config, null); { var ingestDoc = createIngestDoc(Map.of("to_redact", "This is ok nothing to redact")); @@ -171,7 +171,7 @@ public void testRedactWithPatternNamesRedacted() throws Exception { config.put("field", "to_redact"); config.put("patterns", List.of("%{EMAILADDRESS:REDACTED}", "%{IP:REDACTED}", "%{CREDIT_CARD:REDACTED}")); config.put("pattern_definitions", Map.of("CREDIT_CARD", "\\d{4}[ -]\\d{4}[ -]\\d{4}[ -]\\d{4}")); - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config, null); { var ingestDoc = createIngestDoc(Map.of("to_redact", "look a credit card number! 0001-0002-0003-0004 from david@email.com")); @@ -188,7 +188,7 @@ public void testDifferentStartAndEnd() throws Exception { config.put("prefix", "?--"); config.put("suffix", "}"); - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config, null); var ingestDoc = createIngestDoc(Map.of("to_redact", "0.0.0.1 will be redacted")); var redacted = processor.execute(ingestDoc); assertEquals("?--IP_ADDRESS} will be redacted", redacted.getFieldValue("to_redact", String.class)); @@ -199,7 +199,7 @@ public void testDifferentStartAndEnd() throws Exception { config.put("patterns", List.of("%{IP:IP_ADDRESS}")); config.put("prefix", "?--"); - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config, null); var ingestDoc = createIngestDoc(Map.of("to_redact", "0.0.0.1 will be redacted")); var redacted = processor.execute(ingestDoc); assertEquals("?--IP_ADDRESS> will be redacted", redacted.getFieldValue("to_redact", String.class)); @@ -210,7 +210,7 @@ public void testDifferentStartAndEnd() throws Exception { config.put("patterns", List.of("%{IP:IP_ADDRESS}")); config.put("suffix", "++"); - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config, null); var ingestDoc = createIngestDoc(Map.of("to_redact", "0.0.0.1 will be redacted")); var redacted = processor.execute(ingestDoc); assertEquals("(); config.put("field", "to_redact"); config.put("patterns", List.of("foo")); - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config, null); var ingestDoc = createIngestDoc(Map.of("not_the_field", "fieldValue")); var processed = processor.execute(ingestDoc); assertThat(ingestDoc, sameInstance(processed)); @@ -234,7 +234,7 @@ public void testIgnoreMissing() throws Exception { config.put("patterns", List.of("foo")); config.put("ignore_missing", false); // this time the missing field should error - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config, null); var ingestDoc = createIngestDoc(Map.of("not_the_field", "fieldValue")); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDoc)); assertThat(e.getMessage(), containsString("field [to_redact] is null or missing")); @@ -249,7 +249,7 @@ public void testLicenseChecks() throws Exception { config.put("patterns", List.of("foo")); config.put("ignore_missing", false); // usually, this would throw, but here it doesn't because of the license check config.put("skip_if_unlicensed", true); // set the value to true (versus using the default, which is false) - var processor = new RedactProcessor.Factory(notAllowed, MatcherWatchdog.noop()).create(null, "t", "d", config); + var processor = new RedactProcessor.Factory(notAllowed, MatcherWatchdog.noop()).create(null, "t", "d", config, null); assertThat(processor.getSkipIfUnlicensed(), equalTo(true)); var ingestDoc = createIngestDoc(Map.of("not_the_field", "fieldValue")); @@ -299,7 +299,7 @@ public void testLicenseChanges() throws Exception { // constructing the processor is allowed, including extraValidation RedactProcessor.Factory factory = new RedactProcessor.Factory(licenseState, MatcherWatchdog.noop()); - RedactProcessor processor = factory.create(null, null, null, config); + RedactProcessor processor = factory.create(null, null, null, config, null); processor.extraValidation(); // it works great as long as the feature is allowed for the license @@ -338,7 +338,8 @@ public void testTraceRedact() throws Exception { null, "t", "d", - new HashMap<>(config) + new HashMap<>(config), + null ); var message = "this should not be redacted"; var ingestDoc = createIngestDoc(Map.of("to_redact", message)); @@ -352,7 +353,8 @@ public void testTraceRedact() throws Exception { null, "t", "d", - new HashMap<>(config) + new HashMap<>(config), + null ); var ingestDoc = createIngestDoc(Map.of("to_redact", "thisisanemail@address.com will be redacted")); var redactedDoc = processor.execute(ingestDoc); @@ -372,7 +374,13 @@ public void testTraceRedact() throws Exception { configNoTrace.put("field", "to_redact"); configNoTrace.put("patterns", List.of("%{EMAILADDRESS:REDACTED}")); - var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", configNoTrace); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create( + null, + "t", + "d", + configNoTrace, + null + ); var ingestDoc = createIngestDoc(Map.of("to_redact", "thisisanemail@address.com will be redacted")); var redactedDoc = processor.execute(ingestDoc); @@ -398,13 +406,15 @@ public void testTraceRedactMultipleProcessors() throws Exception { null, "t1", "d", - new HashMap<>(configRedact) + new HashMap<>(configRedact), + null ); var processorNoRedact = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create( null, "t2", "d", - new HashMap<>(configNoRedact) + new HashMap<>(configNoRedact), + null ); var ingestDocWithEmail = createIngestDoc(Map.of("to_redact", "thisisanemail@address.com will be redacted")); @@ -420,13 +430,15 @@ public void testTraceRedactMultipleProcessors() throws Exception { null, "t1", "d", - new HashMap<>(configRedact) + new HashMap<>(configRedact), + null ); var processorNoRedact = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create( null, "t2", "d", - new HashMap<>(configNoRedact) + new HashMap<>(configNoRedact), + null ); var ingestDocWithEmail = createIngestDoc(Map.of("to_redact", "thisisanemail@address.com will be redacted")); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java index 62a1a1466ec84..6445bd6ed3010 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; @@ -229,7 +230,8 @@ public SetSecurityUserProcessor create( Map processorFactories, String tag, String description, - Map config + Map config, + ProjectId projectId ) throws Exception { String field = readStringProperty(TYPE, tag, config, "field"); List propertyNames = readOptionalList(TYPE, tag, config, "properties"); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java index d3a6cafbbdd3f..515b5fafa9fc2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java @@ -36,7 +36,7 @@ public void testProcessor() throws Exception { SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> securityContext, Settings.EMPTY); Map config = new HashMap<>(); config.put("field", "_field"); - SetSecurityUserProcessor processor = factory.create(null, "_tag", null, config); + SetSecurityUserProcessor processor = factory.create(null, "_tag", null, config, null); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(EnumSet.allOf(Property.class))); } @@ -44,7 +44,10 @@ public void testProcessor() throws Exception { public void testProcessor_noField() throws Exception { SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> securityContext, Settings.EMPTY); Map config = new HashMap<>(); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(null, "_tag", null, config, null) + ); assertThat(e.getMetadata("es.property_name").get(0), equalTo("field")); assertThat(e.getMetadata("es.processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE)); assertThat(e.getMetadata("es.processor_tag").get(0), equalTo("_tag")); @@ -55,7 +58,7 @@ public void testProcessor_validProperties() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("properties", Arrays.asList(Property.USERNAME.name(), Property.ROLES.name())); - SetSecurityUserProcessor processor = factory.create(null, "_tag", null, config); + SetSecurityUserProcessor processor = factory.create(null, "_tag", null, config, null); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(EnumSet.of(Property.USERNAME, Property.ROLES))); } @@ -65,7 +68,10 @@ public void testProcessor_invalidProperties() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("properties", Arrays.asList("invalid")); - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config)); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> factory.create(null, "_tag", null, config, null) + ); assertThat(e.getMetadata("es.property_name").get(0), equalTo("properties")); assertThat(e.getMetadata("es.processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE)); assertThat(e.getMetadata("es.processor_tag").get(0), equalTo("_tag")); @@ -76,7 +82,7 @@ public void testCanConstructorProcessorWithoutSecurityEnabled() throws Exception SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(() -> null, securityDisabled); Map config = new HashMap<>(); config.put("field", "_field"); - final SetSecurityUserProcessor processor = factory.create(null, "_tag", null, config); + final SetSecurityUserProcessor processor = factory.create(null, "_tag", null, config, null); assertThat(processor, notNullValue()); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessor.java index cc788a45ee91e..7ad0d668b208e 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessor.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessor.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.spatial.ingest; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.GeometryParserFormat; import org.elasticsearch.common.xcontent.XContentHelper; @@ -140,7 +141,8 @@ public CircleProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field); diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessor.java index 4483ea382a820..13e1d614b02fe 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessor.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessor.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.spatial.ingest; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.GeometryParserFormat; import org.elasticsearch.common.xcontent.XContentHelper; @@ -318,7 +319,8 @@ public GeoGridProcessor create( Map registry, String processorTag, String description, - Map config + Map config, + ProjectId projectId ) { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorFactoryTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorFactoryTests.java index 9c036b68e3acc..92c7f204fdd66 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorFactoryTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorFactoryTests.java @@ -30,7 +30,7 @@ public void testCreateGeoShape() { config.put("error_distance", 0.002); config.put("shape_type", "geo_shape"); String processorTag = randomAlphaOfLength(10); - CircleProcessor processor = factory.create(null, processorTag, null, config); + CircleProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.field(), equalTo("field1")); assertThat(processor.targetField(), equalTo("field1")); @@ -44,7 +44,7 @@ public void testCreateShape() { config.put("error_distance", 0.002); config.put("shape_type", "shape"); String processorTag = randomAlphaOfLength(10); - CircleProcessor processor = factory.create(null, processorTag, null, config); + CircleProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.field(), equalTo("field1")); assertThat(processor.targetField(), equalTo("field1")); @@ -58,7 +58,10 @@ public void testCreateInvalidShapeType() { config.put("error_distance", 0.002); config.put("shape_type", "invalid"); String processorTag = randomAlphaOfLength(10); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, processorTag, null, config)); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> factory.create(null, processorTag, null, config, null) + ); assertThat(e.getMessage(), equalTo("illegal [shape_type] value [invalid]. valid values are [SHAPE, GEO_SHAPE]")); } @@ -67,7 +70,7 @@ public void testCreateMissingField() { String processorTag = randomAlphaOfLength(10); ElasticsearchParseException e = expectThrows( ElasticsearchParseException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(e.getMessage(), equalTo("[field] required property is missing")); } @@ -79,7 +82,7 @@ public void testCreateWithTargetField() { config.put("error_distance", 0.002); config.put("shape_type", "geo_shape"); String processorTag = randomAlphaOfLength(10); - CircleProcessor processor = factory.create(null, processorTag, null, config); + CircleProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.field(), equalTo("field1")); assertThat(processor.targetField(), equalTo("other")); @@ -93,7 +96,7 @@ public void testCreateWithNoErrorDistanceDefined() { String processorTag = randomAlphaOfLength(10); ElasticsearchParseException e = expectThrows( ElasticsearchParseException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(e.getMessage(), equalTo("[error_distance] required property is missing")); } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessorFactoryTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessorFactoryTests.java index 89d49fe9104f0..97625efaa9fe5 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessorFactoryTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessorFactoryTests.java @@ -30,7 +30,7 @@ public void testCreateGeohash() { config.put("field", "field1"); config.put("tile_type", GeoGridProcessor.TileFieldType.GEOHASH.name()); String processorTag = randomAlphaOfLength(10); - GeoGridProcessor processor = factory.create(null, processorTag, null, config); + GeoGridProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertFields(processor, "field1", "field1"); assertThat(processor.tileType(), equalTo(GeoGridProcessor.TileFieldType.GEOHASH)); @@ -42,7 +42,7 @@ public void testCreateGeoTile() { config.put("field", "field1"); config.put("tile_type", GeoGridProcessor.TileFieldType.GEOTILE.name()); String processorTag = randomAlphaOfLength(10); - GeoGridProcessor processor = factory.create(null, processorTag, null, config); + GeoGridProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertFields(processor, "field1", "field1"); assertThat(processor.tileType(), equalTo(GeoGridProcessor.TileFieldType.GEOTILE)); @@ -54,7 +54,7 @@ public void testCreateGeohex() { config.put("field", "field1"); config.put("tile_type", GeoGridProcessor.TileFieldType.GEOHEX.name()); String processorTag = randomAlphaOfLength(10); - GeoGridProcessor processor = factory.create(null, processorTag, null, config); + GeoGridProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertFields(processor, "field1", "field1"); assertThat(processor.tileType(), equalTo(GeoGridProcessor.TileFieldType.GEOHEX)); @@ -69,7 +69,7 @@ public void testCreateInvalidTargetFormat() { String processorTag = randomAlphaOfLength(10); ElasticsearchParseException e = expectThrows( ElasticsearchParseException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(e.getMessage(), equalTo("[target_format] illegal value [invalid], valid values are [WKT, GEOJSON]")); } @@ -79,7 +79,7 @@ public void testCreateMissingField() { String processorTag = randomAlphaOfLength(10); ElasticsearchParseException e = expectThrows( ElasticsearchParseException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(e.getMessage(), equalTo("[field] required property is missing")); } @@ -91,7 +91,7 @@ public void testCreateWithTargetField() { config.put("tile_type", GeoGridProcessor.TileFieldType.GEOTILE.name()); config.put("target_format", "wkt"); String processorTag = randomAlphaOfLength(10); - GeoGridProcessor processor = factory.create(null, processorTag, null, config); + GeoGridProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertFields(processor, "field1", "other"); assertThat(processor.tileType(), equalTo(GeoGridProcessor.TileFieldType.GEOTILE)); @@ -104,7 +104,7 @@ public void testCreateWithChildrenField() { config.put("tile_type", GeoGridProcessor.TileFieldType.GEOHEX.name()); config.put("children_field", "children"); String processorTag = randomAlphaOfLength(10); - GeoGridProcessor processor = factory.create(null, processorTag, null, config); + GeoGridProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertFields(processor, "field1", "field1", "", "children", "", ""); assertThat(processor.tileType(), equalTo(GeoGridProcessor.TileFieldType.GEOHEX)); @@ -121,7 +121,7 @@ public void testCreateWithExtraFields() { config.put("precision_field", "precision"); config.put("target_format", "wkt"); String processorTag = randomAlphaOfLength(10); - GeoGridProcessor processor = factory.create(null, processorTag, null, config); + GeoGridProcessor processor = factory.create(null, processorTag, null, config, null); assertThat(processor.getTag(), equalTo(processorTag)); assertFields(processor, "field1", "field1", "parent", "children", "nonChildren", "precision"); assertThat(processor.tileType(), equalTo(GeoGridProcessor.TileFieldType.GEOHEX)); @@ -134,7 +134,7 @@ public void testCreateWithNoTileTypeDefined() { String processorTag = randomAlphaOfLength(10); ElasticsearchParseException e = expectThrows( ElasticsearchParseException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(e.getMessage(), equalTo("[tile_type] required property is missing")); } @@ -146,7 +146,7 @@ public void testCreateWithInvalidTileTypeDefined() { String processorTag = randomAlphaOfLength(10); ElasticsearchParseException e = expectThrows( ElasticsearchParseException.class, - () -> factory.create(null, processorTag, null, config) + () -> factory.create(null, processorTag, null, config, null) ); assertThat(e.getMessage(), equalTo("[tile_type] illegal value [super_tiles], valid values are [GEOHASH, GEOTILE, GEOHEX]")); } diff --git a/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle b/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle index e24580f9af746..0f18b99e549f2 100644 --- a/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle +++ b/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle @@ -34,11 +34,6 @@ tasks.named("yamlRestTest").configure { '^data_streams/10_data_stream_resolvability/*', '^deprecation/10_basic/*', '^dlm/10_usage/*', - '^enrich/10_basic/*', - '^enrich/20_standard_index/*', - '^enrich/30_tsdb_index/*', - '^enrich/40_synthetic_source/*', - '^enrich/50_data_stream/*', '^esql/46_downsample/*', '^esql/60_enrich/*', '^esql/60_usage/*',