Skip to content

Commit 0b9558f

Browse files
committed
ingest pipeline reroute: allow external inspection/reset of state
Logstash's Integration filter works directly with the processors, but cannot use the IngestService that is tightly-coupled with cluster state and must therefore emulate the behavior introduced in elastic#94000. To do so, the additional methods for inquiring about and resetting the reroute state need to be externally-accessible.
1 parent 2f93cf5 commit 0b9558f

File tree

2 files changed

+23
-2
lines changed

2 files changed

+23
-2
lines changed

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorTests.java

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import java.util.Map;
2121

2222
import static org.hamcrest.Matchers.equalTo;
23+
import static org.hamcrest.Matchers.is;
2324

2425
public class RerouteProcessorTests extends ESTestCase {
2526

@@ -29,6 +30,7 @@ public void testDefaults() throws Exception {
2930
RerouteProcessor processor = createRerouteProcessor(List.of(), List.of());
3031
processor.execute(ingestDocument);
3132
assertDataSetFields(ingestDocument, "logs", "generic", "default");
33+
assertThat(ingestDocument.isReroute(), is(true));
3234
}
3335

3436
public void testEventDataset() throws Exception {
@@ -38,6 +40,7 @@ public void testEventDataset() throws Exception {
3840
RerouteProcessor processor = createRerouteProcessor(List.of("{{event.dataset}}"), List.of());
3941
processor.execute(ingestDocument);
4042
assertDataSetFields(ingestDocument, "logs", "foo", "default");
43+
assertThat(ingestDocument.isReroute(), is(true));
4144
assertThat(ingestDocument.getFieldValue("event.dataset", String.class), equalTo("foo"));
4245
}
4346

@@ -48,6 +51,7 @@ public void testEventDatasetDottedFieldName() throws Exception {
4851
RerouteProcessor processor = createRerouteProcessor(List.of("{{event.dataset}}"), List.of());
4952
processor.execute(ingestDocument);
5053
assertDataSetFields(ingestDocument, "logs", "foo", "default");
54+
assertThat(ingestDocument.isReroute(), is(true));
5155
assertThat(ingestDocument.getCtxMap().get("event.dataset"), equalTo("foo"));
5256
assertFalse(ingestDocument.getCtxMap().containsKey("event"));
5357
}
@@ -59,6 +63,7 @@ public void testNoDataset() throws Exception {
5963
RerouteProcessor processor = createRerouteProcessor(List.of("{{ds}}"), List.of());
6064
processor.execute(ingestDocument);
6165
assertDataSetFields(ingestDocument, "logs", "foo", "default");
66+
assertThat(ingestDocument.isReroute(), is(true));
6267
assertFalse(ingestDocument.hasField("event.dataset"));
6368
}
6469

@@ -70,6 +75,7 @@ public void testSkipFirstProcessor() throws Exception {
7075
CompoundProcessor processor = new CompoundProcessor(new SkipProcessor(skippedProcessor), executedProcessor);
7176
processor.execute(ingestDocument);
7277
assertDataSetFields(ingestDocument, "logs", "executed", "default");
78+
assertThat(ingestDocument.isReroute(), is(true));
7379
}
7480

7581
public void testSkipLastProcessor() throws Exception {
@@ -80,6 +86,7 @@ public void testSkipLastProcessor() throws Exception {
8086
CompoundProcessor processor = new CompoundProcessor(executedProcessor, skippedProcessor);
8187
processor.execute(ingestDocument);
8288
assertDataSetFields(ingestDocument, "logs", "executed", "default");
89+
assertThat(ingestDocument.isReroute(), is(true));
8390
}
8491

8592
public void testDataStreamFieldsFromDocument() throws Exception {
@@ -90,6 +97,7 @@ public void testDataStreamFieldsFromDocument() throws Exception {
9097
RerouteProcessor processor = createRerouteProcessor(List.of(), List.of());
9198
processor.execute(ingestDocument);
9299
assertDataSetFields(ingestDocument, "logs", "foo", "bar");
100+
assertThat(ingestDocument.isReroute(), is(true));
93101
}
94102

95103
public void testDataStreamFieldsFromDocumentDottedNotation() throws Exception {
@@ -101,6 +109,7 @@ public void testDataStreamFieldsFromDocumentDottedNotation() throws Exception {
101109
RerouteProcessor processor = createRerouteProcessor(List.of(), List.of());
102110
processor.execute(ingestDocument);
103111
assertDataSetFields(ingestDocument, "logs", "foo", "bar");
112+
assertThat(ingestDocument.isReroute(), is(true));
104113
}
105114

106115
public void testInvalidDataStreamFieldsFromDocument() throws Exception {
@@ -111,6 +120,7 @@ public void testInvalidDataStreamFieldsFromDocument() throws Exception {
111120
RerouteProcessor processor = createRerouteProcessor(List.of(), List.of());
112121
processor.execute(ingestDocument);
113122
assertDataSetFields(ingestDocument, "logs", "foo_bar", "baz_qux");
123+
assertThat(ingestDocument.isReroute(), is(true));
114124
}
115125

116126
public void testDestination() throws Exception {
@@ -120,6 +130,7 @@ public void testDestination() throws Exception {
120130
processor.execute(ingestDocument);
121131
assertFalse(ingestDocument.hasField("data_stream"));
122132
assertThat(ingestDocument.getFieldValue("_index", String.class), equalTo("foo"));
133+
assertThat(ingestDocument.isReroute(), is(true));
123134
}
124135

125136
public void testFieldReference() throws Exception {
@@ -130,6 +141,7 @@ public void testFieldReference() throws Exception {
130141
RerouteProcessor processor = createRerouteProcessor(List.of("{{service.name}}"), List.of("{{service.environment}}"));
131142
processor.execute(ingestDocument);
132143
assertDataSetFields(ingestDocument, "logs", "opbeans_java", "dev");
144+
assertThat(ingestDocument.isReroute(), is(true));
133145
}
134146

135147
public void testRerouteToCurrentTarget() throws Exception {
@@ -142,6 +154,7 @@ public void testRerouteToCurrentTarget() throws Exception {
142154
);
143155
processor.execute(ingestDocument);
144156
assertDataSetFields(ingestDocument, "logs", "generic", "default");
157+
assertThat(ingestDocument.isReroute(), is(true));
145158
assertFalse(ingestDocument.hasField("pipeline_is_continued"));
146159
}
147160

@@ -156,6 +169,7 @@ public void testFieldReferenceWithMissingReroutesToCurrentTarget() throws Except
156169
processor.execute(ingestDocument);
157170
assertThat(ingestDocument.getFieldValue("_index", String.class), equalTo("logs-generic-default"));
158171
assertDataSetFields(ingestDocument, "logs", "generic", "default");
172+
assertThat(ingestDocument.isReroute(), is(true));
159173
assertFalse(ingestDocument.hasField("pipeline_is_continued"));
160174
}
161175

@@ -170,6 +184,7 @@ public void testDataStreamFieldReference() throws Exception {
170184
);
171185
processor.execute(ingestDocument);
172186
assertDataSetFields(ingestDocument, "logs", "dataset_from_doc", "namespace_from_doc");
187+
assertThat(ingestDocument.isReroute(), is(true));
173188
}
174189

175190
public void testDatasetFieldReferenceMissingValue() throws Exception {
@@ -181,6 +196,7 @@ public void testDatasetFieldReferenceMissingValue() throws Exception {
181196
);
182197
processor.execute(ingestDocument);
183198
assertDataSetFields(ingestDocument, "logs", "fallback", "fallback");
199+
assertThat(ingestDocument.isReroute(), is(true));
184200
}
185201

186202
public void testDatasetFieldReference() throws Exception {
@@ -194,6 +210,7 @@ public void testDatasetFieldReference() throws Exception {
194210
);
195211
processor.execute(ingestDocument);
196212
assertDataSetFields(ingestDocument, "logs", "generic", "default");
213+
assertThat(ingestDocument.isReroute(), is(true));
197214
}
198215

199216
public void testFallbackToValuesFrom_index() throws Exception {
@@ -204,6 +221,7 @@ public void testFallbackToValuesFrom_index() throws Exception {
204221
RerouteProcessor processor = createRerouteProcessor(List.of("{{foo}}"), List.of("{{bar}}"));
205222
processor.execute(ingestDocument);
206223
assertDataSetFields(ingestDocument, "logs", "generic", "default");
224+
assertThat(ingestDocument.isReroute(), is(true));
207225
}
208226

209227
public void testInvalidDataStreamName() throws Exception {
@@ -212,6 +230,7 @@ public void testInvalidDataStreamName() throws Exception {
212230
RerouteProcessor processor = createRerouteProcessor(List.of(), List.of());
213231
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
214232
assertThat(e.getMessage(), equalTo("invalid data stream name: [foo]; must follow naming scheme <type>-<dataset>-<namespace>"));
233+
assertThat(ingestDocument.isReroute(), is(false));
215234
}
216235

217236
{
@@ -220,6 +239,7 @@ public void testInvalidDataStreamName() throws Exception {
220239
RerouteProcessor processor = createRerouteProcessor("bar");
221240
processor.execute(ingestDocument);
222241
assertThat(ingestDocument.getFieldValue("_index", String.class), equalTo("bar"));
242+
assertThat(ingestDocument.isReroute(), is(true));
223243
}
224244
}
225245

@@ -229,6 +249,7 @@ public void testRouteOnNonStringFieldFails() {
229249
RerouteProcessor processor = createRerouteProcessor(List.of("{{numeric_field}}"), List.of());
230250
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
231251
assertThat(e.getMessage(), equalTo("field [numeric_field] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
252+
assertThat(ingestDocument.isReroute(), is(false));
232253
}
233254

234255
public void testDatasetSanitization() {

server/src/main/java/org/elasticsearch/ingest/IngestDocument.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -932,15 +932,15 @@ public void reroute(String destIndex) {
932932
*
933933
* @return whether the document is redirected to another target
934934
*/
935-
boolean isReroute() {
935+
public boolean isReroute() {
936936
return reroute;
937937
}
938938

939939
/**
940940
* Set the {@link #reroute} flag to false so that subsequent calls to {@link #isReroute()} will return false until/unless
941941
* {@link #reroute(String)} is called.
942942
*/
943-
void resetReroute() {
943+
public void resetReroute() {
944944
reroute = false;
945945
}
946946

0 commit comments

Comments
 (0)