Skip to content

Commit 37807b8

Browse files
martijnvgmartijnvg
authored
Support flattened label field with downsampling. (#118816)
If flattened field is configured as non-dimension and non-metric field, then downsampling fails to execute successfully. Downsampling doesn't know how to use the flattened field or how to serialize it. This change addresses this. Closes #116319
1 parent c75e92a commit 37807b8

File tree

7 files changed

+428
-9
lines changed

7 files changed

+428
-9
lines changed

‎docs/changelog/118816.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 118816
2+
summary: Support flattened field with downsampling
3+
area: Downsampling
4+
type: bug
5+
issues:
6+
- 116319

‎server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import org.elasticsearch.index.mapper.DocumentParserContext;
5353
import org.elasticsearch.index.mapper.DynamicFieldType;
5454
import org.elasticsearch.index.mapper.FieldMapper;
55+
import org.elasticsearch.index.mapper.KeywordFieldMapper;
5556
import org.elasticsearch.index.mapper.MappedFieldType;
5657
import org.elasticsearch.index.mapper.Mapper;
5758
import org.elasticsearch.index.mapper.MapperBuilderContext;
@@ -670,7 +671,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme
670671
private final boolean isDimension;
671672
private final int ignoreAbove;
672673

673-
public RootFlattenedFieldType(
674+
RootFlattenedFieldType(
674675
String name,
675676
boolean indexed,
676677
boolean hasDocValues,
@@ -682,7 +683,7 @@ public RootFlattenedFieldType(
682683
this(name, indexed, hasDocValues, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList(), ignoreAbove);
683684
}
684685

685-
public RootFlattenedFieldType(
686+
RootFlattenedFieldType(
686687
String name,
687688
boolean indexed,
688689
boolean hasDocValues,
@@ -806,6 +807,10 @@ public MappedFieldType getChildFieldType(String childPath) {
806807
return new KeyedFlattenedFieldType(name(), childPath, this);
807808
}
808809

810+
public MappedFieldType getKeyedFieldType() {
811+
return new KeywordFieldMapper.KeywordFieldType(name() + KEYED_FIELD_SUFFIX);
812+
}
813+
809814
@Override
810815
public boolean isDimension() {
811816
return isDimension;

‎server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@
5555
* }`
5656
*
5757
*/
58-
class FlattenedFieldSyntheticWriterHelper {
58+
public class FlattenedFieldSyntheticWriterHelper {
5959

6060
private record Prefix(List<String> prefix) {
6161

@@ -225,17 +225,17 @@ public boolean equals(Object obj) {
225225
}
226226
}
227227

228-
interface SortedKeyedValues {
228+
public interface SortedKeyedValues {
229229
BytesRef next() throws IOException;
230230
}
231231

232232
private final SortedKeyedValues sortedKeyedValues;
233233

234-
FlattenedFieldSyntheticWriterHelper(final SortedKeyedValues sortedKeyedValues) {
234+
public FlattenedFieldSyntheticWriterHelper(final SortedKeyedValues sortedKeyedValues) {
235235
this.sortedKeyedValues = sortedKeyedValues;
236236
}
237237

238-
void write(final XContentBuilder b) throws IOException {
238+
public void write(final XContentBuilder b) throws IOException {
239239
KeyValue curr = new KeyValue(sortedKeyedValues.next());
240240
KeyValue prev = KeyValue.EMPTY;
241241
final List<String> values = new ArrayList<>();

‎x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/70_flattened_field_type.yml‎

Lines changed: 307 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,307 @@
1+
---
2+
"A flattened label field":
3+
- do:
4+
indices.create:
5+
index: source_index
6+
body:
7+
settings:
8+
number_of_shards: 1
9+
index:
10+
mode: time_series
11+
routing_path: [ metricset, k8s.pod.uid ]
12+
time_series:
13+
start_time: 2021-04-28T00:00:00Z
14+
end_time: 2021-04-29T00:00:00Z
15+
mappings:
16+
subobjects: false
17+
properties:
18+
"@timestamp":
19+
type: date
20+
metricset:
21+
type: keyword
22+
time_series_dimension: true
23+
k8s:
24+
properties:
25+
pod:
26+
properties:
27+
uid:
28+
type: keyword
29+
time_series_dimension: true
30+
name:
31+
type: keyword
32+
agent:
33+
type: flattened
34+
value:
35+
type: long
36+
time_series_metric: gauge
37+
38+
- do:
39+
bulk:
40+
refresh: true
41+
index: source_index
42+
body:
43+
- '{"index": {}}'
44+
- '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.4" }, "value": 10 }}'
45+
- '{"index": {}}'
46+
- '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.5" }, "value": 20 }}'
47+
- '{"index": {}}'
48+
- '{"@timestamp": "2021-04-28T20:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.6" }, "value": 12 }}'
49+
- '{"index": {}}'
50+
- '{"@timestamp": "2021-04-28T20:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.7" }, "value": 15 }}'
51+
- '{"index": {}}'
52+
- '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.7" }, "value": 9 }}'
53+
- '{"index": {}}'
54+
- '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.8" }, "value": 16 }}'
55+
- '{"index": {}}'
56+
- '{"@timestamp": "2021-04-28T19:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.9" }, "value": 25 }}'
57+
- '{"index": {}}'
58+
- '{"@timestamp": "2021-04-28T19:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.10" }, "value": 17 }}'
59+
60+
- do:
61+
indices.put_settings:
62+
index: source_index
63+
body:
64+
index.blocks.write: true
65+
66+
- do:
67+
indices.downsample:
68+
index: source_index
69+
target_index: target_index
70+
body: >
71+
{
72+
"fixed_interval": "1h"
73+
}
74+
- is_true: acknowledged
75+
76+
- do:
77+
search:
78+
index: target_index
79+
body:
80+
sort: [ "_tsid", "@timestamp" ]
81+
82+
- length: { hits.hits: 4 }
83+
- match: { hits.hits.0._source._doc_count: 2 }
84+
- match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
85+
- match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
86+
- match: { hits.hits.0._source.k8s\.agent: { "id": "second", "version": "2.1.8" } }
87+
88+
- match: { hits.hits.1._source._doc_count: 2 }
89+
- match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
90+
- match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z }
91+
- match: { hits.hits.1._source.k8s\.agent: { "id": "second", "version": "2.1.10" } }
92+
93+
- match: { hits.hits.2._source._doc_count: 2 }
94+
- match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
95+
- match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
96+
- match: { hits.hits.2._source.k8s\.agent: { "id": "first", "version": "2.0.5" } }
97+
98+
- match: { hits.hits.3._source._doc_count: 2 }
99+
- match: { hits.hits.3._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
100+
- match: { hits.hits.3._source.@timestamp: 2021-04-28T20:00:00.000Z }
101+
- match: { hits.hits.3._source.k8s\.agent: { "id": "first", "version": "2.0.7" } }
102+
103+
---
104+
"A flattened label field with no doc values":
105+
- do:
106+
indices.create:
107+
index: source_index
108+
body:
109+
settings:
110+
number_of_shards: 1
111+
index:
112+
mode: time_series
113+
routing_path: [ metricset, k8s.pod.uid ]
114+
time_series:
115+
start_time: 2021-04-28T00:00:00Z
116+
end_time: 2021-04-29T00:00:00Z
117+
mappings:
118+
subobjects: false
119+
properties:
120+
"@timestamp":
121+
type: date
122+
metricset:
123+
type: keyword
124+
time_series_dimension: true
125+
k8s:
126+
properties:
127+
pod:
128+
properties:
129+
uid:
130+
type: keyword
131+
time_series_dimension: true
132+
name:
133+
type: keyword
134+
agent:
135+
type: flattened
136+
doc_values: false
137+
value:
138+
type: long
139+
time_series_metric: gauge
140+
141+
- do:
142+
bulk:
143+
refresh: true
144+
index: source_index
145+
body:
146+
- '{"index": {}}'
147+
- '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.4" }, "value": 10 }}'
148+
- '{"index": {}}'
149+
- '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.5" }, "value": 20 }}'
150+
- '{"index": {}}'
151+
- '{"@timestamp": "2021-04-28T20:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.6" }, "value": 12 }}'
152+
- '{"index": {}}'
153+
- '{"@timestamp": "2021-04-28T20:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.7" }, "value": 15 }}'
154+
- '{"index": {}}'
155+
- '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.7" }, "value": 9 }}'
156+
- '{"index": {}}'
157+
- '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.8" }, "value": 16 }}'
158+
- '{"index": {}}'
159+
- '{"@timestamp": "2021-04-28T19:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.9" }, "value": 25 }}'
160+
- '{"index": {}}'
161+
- '{"@timestamp": "2021-04-28T19:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.10" }, "value": 17 }}'
162+
163+
- do:
164+
indices.put_settings:
165+
index: source_index
166+
body:
167+
index.blocks.write: true
168+
169+
- do:
170+
indices.downsample:
171+
index: source_index
172+
target_index: target_index
173+
body: >
174+
{
175+
"fixed_interval": "1h"
176+
}
177+
- is_true: acknowledged
178+
179+
- do:
180+
search:
181+
index: target_index
182+
body:
183+
sort: [ "_tsid", "@timestamp" ]
184+
185+
- length: { hits.hits: 4 }
186+
- match: { hits.hits.0._source._doc_count: 2 }
187+
- match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
188+
- match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
189+
- is_false: hits.hits.0._source.k8s\.agent
190+
191+
- match: { hits.hits.1._source._doc_count: 2 }
192+
- match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
193+
- match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z }
194+
- is_false: hits.hits.1._source.k8s\.agent
195+
196+
- match: { hits.hits.2._source._doc_count: 2 }
197+
- match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
198+
- match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
199+
- is_false: hits.hits.2._source.k8s\.agent
200+
201+
- match: { hits.hits.3._source._doc_count: 2 }
202+
- match: { hits.hits.3._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
203+
- match: { hits.hits.3._source.@timestamp: 2021-04-28T20:00:00.000Z }
204+
- is_false: hits.hits.3._source.k8s\.agent
205+
206+
---
207+
"A flattened label field with mixed content":
208+
- do:
209+
indices.create:
210+
index: source_index
211+
body:
212+
settings:
213+
number_of_shards: 1
214+
index:
215+
mode: time_series
216+
routing_path: [ metricset, k8s.pod.uid ]
217+
time_series:
218+
start_time: 2021-04-28T00:00:00Z
219+
end_time: 2021-04-29T00:00:00Z
220+
mappings:
221+
subobjects: false
222+
properties:
223+
"@timestamp":
224+
type: date
225+
metricset:
226+
type: keyword
227+
time_series_dimension: true
228+
k8s:
229+
properties:
230+
pod:
231+
properties:
232+
uid:
233+
type: keyword
234+
time_series_dimension: true
235+
name:
236+
type: keyword
237+
agent:
238+
type: flattened
239+
null_value: my_null_value
240+
value:
241+
type: long
242+
time_series_metric: gauge
243+
244+
- do:
245+
bulk:
246+
refresh: true
247+
index: source_index
248+
body:
249+
- '{"index": {}}'
250+
- '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.4", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11 }, "value": 10 }}'
251+
- '{"index": {}}'
252+
- '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.5", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 20 }}'
253+
- '{"index": {}}'
254+
- '{"@timestamp": "2021-04-28T20:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.6", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 12 }}'
255+
- '{"index": {}}'
256+
- '{"@timestamp": "2021-04-28T20:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.7", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 15 }}'
257+
- '{"index": {}}'
258+
- '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.7", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 9 }}'
259+
- '{"index": {}}'
260+
- '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.8", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 16 }}'
261+
- '{"index": {}}'
262+
- '{"@timestamp": "2021-04-28T19:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.9", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 25 }}'
263+
- '{"index": {}}'
264+
- '{"@timestamp": "2021-04-28T19:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.10", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 17 }}'
265+
266+
- do:
267+
indices.put_settings:
268+
index: source_index
269+
body:
270+
index.blocks.write: true
271+
272+
- do:
273+
indices.downsample:
274+
index: source_index
275+
target_index: target_index
276+
body: >
277+
{
278+
"fixed_interval": "1h"
279+
}
280+
- is_true: acknowledged
281+
282+
- do:
283+
search:
284+
index: target_index
285+
body:
286+
sort: [ "_tsid", "@timestamp" ]
287+
288+
- length: { hits.hits: 4 }
289+
- match: { hits.hits.0._source._doc_count: 2 }
290+
- match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
291+
- match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
292+
- match: { hits.hits.0._source.k8s\.agent: { "id": "second", "version": "2.1.8", "versions": ["1", "2", "3"], "dotted": {"version": "1.1"}, "numeric_version": "11", optional_version: "my_null_value" } }
293+
294+
- match: { hits.hits.1._source._doc_count: 2 }
295+
- match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
296+
- match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z }
297+
- match: { hits.hits.1._source.k8s\.agent: { "id": "second", "version": "2.1.10", "versions": ["1", "2", "3"], "dotted": {"version": "1.1"}, "numeric_version": "11", optional_version: "my_null_value" } }
298+
299+
- match: { hits.hits.2._source._doc_count: 2 }
300+
- match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
301+
- match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
302+
- match: { hits.hits.2._source.k8s\.agent: { "id": "first", "version": "2.0.5", "versions": ["1", "2", "3"], "dotted": {"version": "1.1"}, "numeric_version": "11", optional_version: "my_null_value" } }
303+
304+
- match: { hits.hits.3._source._doc_count: 2 }
305+
- match: { hits.hits.3._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
306+
- match: { hits.hits.3._source.@timestamp: 2021-04-28T20:00:00.000Z }
307+
- match: { hits.hits.3._source.k8s\.agent: { "id": "first", "version": "2.0.7", "versions": ["1", "2", "3"], "dotted": {"version": "1.1"}, "numeric_version": "11", optional_version: "my_null_value" } }

0 commit comments

Comments
 (0)