Skip to content

Commit e16c1eb

Browse files
DaveCTurnerDaveCTurner
authored
Merge branch 'main' into 2025/09/09/cluster-applier-thread-watchdog
2 parents 8bc49e2 + 7d0ee1a commit e16c1eb

File tree

13 files changed

+223
-18
lines changed

13 files changed

+223
-18
lines changed

‎docs/changelog/134319.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 134319
2+
summary: Fix `allow_duplicates` edge case bug in append processor
3+
area: Ingest Node
4+
type: bug
5+
issues: []

‎docs/reference/query-languages/esql/images/functions/decay.svg‎

Lines changed: 1 addition & 1 deletion
Loading

‎docs/reference/query-languages/esql/images/functions/knn.svg‎

Lines changed: 1 addition & 1 deletion
Loading

‎docs/reference/search-connectors/es-connectors-servicenow.md‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -119,9 +119,6 @@ The ServiceNow connector is compatible with the following versions of ServiceNow
119119

120120
### Configuration [es-connectors-servicenow-client-configuration]
121121

122-
123-
124-
125122
The following configuration fields are required to set up the connector:
126123

127124
`url`
@@ -136,11 +133,11 @@ The following configuration fields are required to set up the connector:
136133
`services`
137134
: Comma-separated list of services to fetch data from ServiceNow. If the value is `*`, the connector will fetch data from the list of basic services provided by ServiceNow:
138135

139-
* [User](https://docs.servicenow.com/bundle/utah-platform-administration/page/administer/roles/concept/user.md)
140-
* [Incident](https://docs.servicenow.com/bundle/tokyo-it-service-management/page/product/incident-management/concept/c_IncidentManagement.md)
141-
* [Requested Item](https://docs.servicenow.com/bundle/tokyo-servicenow-platform/page/use/service-catalog-requests/task/t_AddNewRequestItems.md)
142-
* [Knowledge](https://docs.servicenow.com/bundle/tokyo-customer-service-management/page/product/customer-service-management/task/t_SearchTheKnowledgeBase.md)
143-
* [Change request](https://docs.servicenow.com/bundle/tokyo-it-service-management/page/product/change-management/task/t_CreateAChange.md)
136+
* [User](https://www.servicenow.com/docs/bundle/vancouver-platform-administration/page/administer/roles/concept/user.html)
137+
* [Incident](https://www.servicenow.com/docs/bundle/vancouver-it-service-management/page/product/incident-management/concept/c_IncidentManagement.html)
138+
* [Requested Item](https://www.servicenow.com/docs/bundle/vancouver-servicenow-platform/page/use/service-catalog-requests/task/t_AddNewRequestItems.html)
139+
* [Knowledge](https://www.servicenow.com/docs/bundle/vancouver-servicenow-platform/page/product/knowledge-management/concept/c_KnowledgeHomepage.html)
140+
* [Change request](https://www.servicenow.com/docs/bundle/vancouver-it-service-management/page/product/change-management/task/t_CreateAChange.html)
144141

145142
::::{note}
146143
If you have configured a custom service, the `*` value will not fetch data from the basic services above by default. In this case you’ll need to mention these service names explicitly.
@@ -373,4 +370,4 @@ See [Security](/reference/search-connectors/es-connectors-security.md).
373370

374371
### Content extraction [es-connectors-servicenow-client-content-extraction]
375372

376-
See [Content extraction](/reference/search-connectors/es-connectors-content-extraction.md).
373+
See [Content extraction](/reference/search-connectors/es-connectors-content-extraction.md).

‎modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/360_append_allow_duplicates.yml‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
---
2+
setup:
3+
- requires:
4+
cluster_features: [ "ingest.append.copy_from" ]
5+
reason: "The copy_from option of the append processor is new"
6+
- do:
7+
ingest.put_pipeline:
8+
id: "test-pipeline-1"
9+
body: >
10+
{
11+
"processors": [
12+
{
13+
"append": {
14+
"if": "ctx?.templating != true",
15+
"field": "dest",
16+
"copy_from": "src",
17+
"allow_duplicates": false
18+
}
19+
},
20+
{
21+
"append": {
22+
"if": "ctx?.templating == true",
23+
"field": "dest",
24+
"value": ["{{three}}", "{{three}}", "{{three}}"],
25+
"allow_duplicates": false
26+
}
27+
},
28+
{
29+
"foreach": {
30+
"description": "templating results in strings, so convert them if necessary",
31+
"field": "dest",
32+
"processor": {
33+
"convert": {
34+
"field": "_ingest._value",
35+
"type": "long"
36+
}
37+
}
38+
}
39+
},
40+
{
41+
"remove": {
42+
"description": "we only care about the dest, so remove everything else",
43+
"keep": ["dest"]
44+
}
45+
}
46+
]
47+
}
48+
- do:
49+
indices.create:
50+
index: "test-some-index"
51+
52+
---
53+
teardown:
54+
- do:
55+
indices.delete:
56+
index: "test-some-index"
57+
ignore_unavailable: true
58+
- do:
59+
ingest.delete_pipeline:
60+
id: "test-pipeline-1"
61+
ignore: 404
62+
63+
---
64+
"allow_duplicates (false) removes duplicates with a present array and copy_from":
65+
- do:
66+
index:
67+
index: test-some-index
68+
id: 1
69+
pipeline: test-pipeline-1
70+
body: >
71+
{
72+
"src": [3, 3, 3],
73+
"dest": [3]
74+
}
75+
76+
- do:
77+
get:
78+
index: test-some-index
79+
id: "1"
80+
- match: { _source.dest: [3] }
81+
82+
---
83+
"allow_duplicates (false) removes duplicates with an empty array and copy_from":
84+
- do:
85+
index:
86+
index: test-some-index
87+
id: 1
88+
pipeline: test-pipeline-1
89+
body: >
90+
{
91+
"src": [3, 3, 3],
92+
"dest": []
93+
}
94+
95+
- do:
96+
get:
97+
index: test-some-index
98+
id: "1"
99+
- match: { _source.dest: [3] }
100+
101+
---
102+
"allow_duplicates (false) removes duplicates with a missing array and copy_from":
103+
- do:
104+
index:
105+
index: test-some-index
106+
id: 1
107+
pipeline: test-pipeline-1
108+
body: >
109+
{
110+
"src": [3, 3, 3]
111+
}
112+
113+
- do:
114+
get:
115+
index: test-some-index
116+
id: "1"
117+
- match: { _source.dest: [3] }
118+
119+
---
120+
"allow_duplicates (false) removes duplicates with a present array and templating":
121+
- do:
122+
index:
123+
index: test-some-index
124+
id: 1
125+
pipeline: test-pipeline-1
126+
body: >
127+
{
128+
"templating": true,
129+
"three": 3,
130+
"dest": ["3"],
131+
"note": "blargh, duplicate removal is based on strings, because of templating"
132+
}
133+
134+
- do:
135+
get:
136+
index: test-some-index
137+
id: "1"
138+
- match: { _source.dest: [3] }
139+
140+
---
141+
"allow_duplicates (false) removes duplicates with an empty array and templating":
142+
- do:
143+
index:
144+
index: test-some-index
145+
id: 1
146+
pipeline: test-pipeline-1
147+
body: >
148+
{
149+
"templating": true,
150+
"three": 3,
151+
"dest": []
152+
}
153+
154+
- do:
155+
get:
156+
index: test-some-index
157+
id: "1"
158+
- match: { _source.dest: [3] }
159+
160+
---
161+
"allow_duplicates (false) removes duplicates with a missing array and templating":
162+
- do:
163+
index:
164+
index: test-some-index
165+
id: 1
166+
pipeline: test-pipeline-1
167+
body: >
168+
{
169+
"templating": true,
170+
"three": 3
171+
}
172+
173+
- do:
174+
get:
175+
index: test-some-index
176+
id: "1"
177+
- match: { _source.dest: [3] }

‎muted-tests.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -507,6 +507,9 @@ tests:
507507
- class: org.elasticsearch.action.support.nodes.TransportNodesActionTests
508508
method: testConcurrentlyCompletionAndCancellation
509509
issue: https://github.com/elastic/elasticsearch/issues/134277
510+
- class: org.elasticsearch.xpack.logsdb.qa.BulkDynamicMappingChallengeRestIT
511+
method: testDateHistogramAggregation
512+
issue: https://github.com/elastic/elasticsearch/issues/134389
510513

511514
# Examples:
512515
#

‎server/src/main/java/org/elasticsearch/ingest/IngestDocument.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -861,7 +861,7 @@ private void setFieldValue(String path, Object value, boolean append, boolean al
861861
Object object = map.getOrDefault(leafKey, NOT_FOUND); // getOrDefault is faster than containsKey + get
862862
if (object == NOT_FOUND) {
863863
List<Object> list = new ArrayList<>();
864-
appendValues(list, value);
864+
appendValues(list, value, allowDuplicates);
865865
map.put(leafKey, list);
866866
} else {
867867
Object list = appendValues(object, value, allowDuplicates);
@@ -879,7 +879,7 @@ private void setFieldValue(String path, Object value, boolean append, boolean al
879879
Object object = map.getOrDefault(leafKey, NOT_FOUND); // getOrDefault is faster than containsKey + get
880880
if (object == NOT_FOUND) {
881881
List<Object> list = new ArrayList<>();
882-
appendValues(list, value);
882+
appendValues(list, value, allowDuplicates);
883883
map.put(leafKey, list);
884884
} else {
885885
Object list = appendValues(object, value, allowDuplicates);
@@ -933,23 +933,25 @@ private static Object appendValues(Object maybeList, Object value, boolean allow
933933
list.add(maybeList);
934934
}
935935
if (allowDuplicates) {
936-
appendValues(list, value);
936+
innerAppendValues(list, value);
937937
return list;
938938
} else {
939939
// if no values were appended due to duplication, return the original object so the ingest document remains unmodified
940-
return appendValuesWithoutDuplicates(list, value) ? list : maybeList;
940+
return innerAppendValuesWithoutDuplicates(list, value) ? list : maybeList;
941941
}
942942
}
943943

944-
private static void appendValues(List<Object> list, Object value) {
944+
// helper method for use in appendValues above, please do not call this directly except from that method
945+
private static void innerAppendValues(List<Object> list, Object value) {
945946
if (value instanceof List<?> l) {
946947
list.addAll(l);
947948
} else {
948949
list.add(value);
949950
}
950951
}
951952

952-
private static boolean appendValuesWithoutDuplicates(List<Object> list, Object value) {
953+
// helper method for use in appendValues above, please do not call this directly except from that method
954+
private static boolean innerAppendValuesWithoutDuplicates(List<Object> list, Object value) {
953955
boolean valuesWereAppended = false;
954956
if (value instanceof List<?> valueList) {
955957
for (Object val : valueList) {

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1329,6 +1329,11 @@ public enum Cap {
13291329
*/
13301330
FIX_MV_EXPAND_INCONSISTENT_COLUMN_ORDER,
13311331

1332+
/**
1333+
* Support for the SET command.
1334+
*/
1335+
SET_COMMAND(Build.current().isSnapshot()),
1336+
13321337
/**
13331338
* (Re)Added EXPLAIN command
13341339
*/

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ public void testInvalidSample() {
213213
}
214214

215215
public void testSet() {
216+
assumeTrue("SET command available in snapshot only", EsqlCapabilities.Cap.SET_COMMAND.isEnabled());
216217
EsqlStatement query = parse("SET foo = \"bar\"; row a = 1", new QueryParams());
217218
assertThat(query.plan(), is(instanceOf(Row.class)));
218219
assertThat(query.settings().size(), is(1));
@@ -232,6 +233,7 @@ public void testSet() {
232233
}
233234

234235
public void testSetWithTripleQuotes() {
236+
assumeTrue("SET command available in snapshot only", EsqlCapabilities.Cap.SET_COMMAND.isEnabled());
235237
EsqlStatement query = parse("SET foo = \"\"\"bar\"baz\"\"\"; row a = 1", new QueryParams());
236238
assertThat(query.plan(), is(instanceOf(Row.class)));
237239
assertThat(query.settings().size(), is(1));
@@ -249,6 +251,7 @@ public void testSetWithTripleQuotes() {
249251
}
250252

251253
public void testMultipleSet() {
254+
assumeTrue("SET command available in snapshot only", EsqlCapabilities.Cap.SET_COMMAND.isEnabled());
252255
EsqlStatement query = parse(
253256
"SET foo = \"bar\"; SET bar = 2; SET foo = \"baz\"; SET x = 3.5; SET y = false; SET z = null; row a = 1",
254257
new QueryParams()
@@ -265,6 +268,7 @@ public void testMultipleSet() {
265268
}
266269

267270
public void testSetArrays() {
271+
assumeTrue("SET command available in snapshot only", EsqlCapabilities.Cap.SET_COMMAND.isEnabled());
268272
EsqlStatement query = parse("SET foo = [\"bar\", \"baz\"]; SET bar = [1, 2, 3]; row a = 1", new QueryParams());
269273
assertThat(query.plan(), is(instanceOf(Row.class)));
270274
assertThat(query.settings().size(), is(2));
@@ -274,6 +278,7 @@ public void testSetArrays() {
274278
}
275279

276280
public void testSetWithNamedParams() {
281+
assumeTrue("SET command available in snapshot only", EsqlCapabilities.Cap.SET_COMMAND.isEnabled());
277282
EsqlStatement query = parse(
278283
"SET foo = \"bar\"; SET bar = ?a; SET foo = \"baz\"; SET x = ?x; row a = 1",
279284
new QueryParams(
@@ -293,6 +298,7 @@ public void testSetWithNamedParams() {
293298
}
294299

295300
public void testSetWithPositionalParams() {
301+
assumeTrue("SET command available in snapshot only", EsqlCapabilities.Cap.SET_COMMAND.isEnabled());
296302
EsqlStatement query = parse(
297303
"SET foo = \"bar\"; SET bar = ?; SET foo = \"baz\"; SET x = ?; row a = ?",
298304
new QueryParams(

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2069,6 +2069,8 @@ public void testCategorizeOptionSimilarityThreshold() {
20692069
}
20702070

20712071
public void testCategorizeWithInlineStats() {
2072+
assumeTrue("CATEGORIZE must be enabled", EsqlCapabilities.Cap.CATEGORIZE_V6.isEnabled());
2073+
assumeTrue("INLINESTATS must be enabled", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled());
20722074
assertEquals(
20732075
"1:37: CATEGORIZE [CATEGORIZE(last_name, { \"similarity_threshold\": 1 })] is not yet supported with "
20742076
+ "INLINESTATS [INLINESTATS COUNT(*) BY CATEGORIZE(last_name, { \"similarity_threshold\": 1 })]",

0 commit comments

Comments
 (0)