Skip to content

Commit 1b79e3a

Browse files
SoumyaRaikwarSoumyaRaikwar
committed
[storage] Add Elasticsearch data stream support
Part of #4708 Adds opt-in support for Elasticsearch data streams to replace traditional date-based indices with automatic lifecycle management. Signed-off-by: SoumyaRaikwar <somuraik@gmail.com>
1 parent c0b1db9 commit 1b79e3a

File tree

9 files changed

+473
-4
lines changed

9 files changed

+473
-4
lines changed

‎internal/storage/elasticsearch/config/config.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,9 @@ type Configuration struct {
168168
// Read more about ILM at
169169
// https://www.jaegertracing.io/docs/deployment/#enabling-ilm-support
170170
UseILM bool `mapstructure:"use_ilm"`
171+
// UseDataStream, if set to true, will use Elasticsearch data streams for storing traces.
172+
// This requires Elasticsearch 7.9+.
173+
UseDataStream bool `mapstructure:"use_data_stream"`
171174

172175
// ---- jaeger-specific configs ----
173176
// MaxDocCount Defines maximum number of results to fetch from storage per query.
@@ -536,6 +539,9 @@ func (c *Configuration) ApplyDefaults(source *Configuration) {
536539
c.CustomHeaders[k] = v
537540
}
538541
}
542+
if !c.UseDataStream {
543+
c.UseDataStream = source.UseDataStream
544+
}
539545
}
540546

541547
// RolloverFrequencyAsNegativeDuration returns the index rollover frequency duration for the given frequency string

‎internal/storage/v1/elasticsearch/factory.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,12 +182,16 @@ func (f *FactoryBase) CreateSamplingStore(int /* maxBuckets */) (samplingstore.S
182182
return store, nil
183183
}
184184

185+
const defaultILMPolicyName = "jaeger-ilm-policy"
186+
185187
func (f *FactoryBase) mappingBuilderFromConfig(cfg *config.Configuration) mappings.MappingBuilder {
186188
return mappings.MappingBuilder{
187189
TemplateBuilder: f.templateBuilder,
188190
Indices: cfg.Indices,
189191
EsVersion: cfg.Version,
190192
UseILM: cfg.UseILM,
193+
UseDataStream: cfg.UseDataStream,
194+
ILMPolicyName: cfg.Indices.IndexPrefix.Apply(defaultILMPolicyName),
191195
}
192196
}
193197

‎internal/storage/v1/elasticsearch/mappings/fixtures/jaeger-span-ds-8.json‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
{
2+
"priority": 500,
3+
"index_patterns": "test-jaeger-span-ds",
4+
"data_stream": {},
5+
"template": {
6+
"settings": {
7+
"index.number_of_shards": 3,
8+
"index.number_of_replicas": 3,
9+
"index.mapping.nested_fields.limit": 50,
10+
"index.requests.cache.enable": true,
11+
"index.default_pipeline": "jaeger-span-ds-timestamp",
12+
"index.lifecycle.name": "jaeger-test-policy"
13+
},
14+
"mappings": {
15+
"dynamic_templates": [
16+
{
17+
"span_tags_map": {
18+
"mapping": {
19+
"type": "keyword",
20+
"ignore_above": 256
21+
},
22+
"path_match": "tag.*"
23+
}
24+
},
25+
{
26+
"process_tags_map": {
27+
"mapping": {
28+
"type": "keyword",
29+
"ignore_above": 256
30+
},
31+
"path_match": "process.tag.*"
32+
}
33+
}
34+
],
35+
"properties": {
36+
"@timestamp": {
37+
"type": "date",
38+
"format": "epoch_millis"
39+
},
40+
"traceID": {
41+
"type": "keyword",
42+
"ignore_above": 256
43+
},
44+
"parentSpanID": {
45+
"type": "keyword",
46+
"ignore_above": 256
47+
},
48+
"spanID": {
49+
"type": "keyword",
50+
"ignore_above": 256
51+
},
52+
"operationName": {
53+
"type": "keyword",
54+
"ignore_above": 256
55+
},
56+
"startTime": {
57+
"type": "long"
58+
},
59+
"startTimeMillis": {
60+
"type": "date",
61+
"format": "epoch_millis"
62+
},
63+
"duration": {
64+
"type": "long"
65+
},
66+
"flags": {
67+
"type": "integer"
68+
},
69+
"logs": {
70+
"type": "nested",
71+
"dynamic": false,
72+
"properties": {
73+
"timestamp": {
74+
"type": "long"
75+
},
76+
"fields": {
77+
"type": "nested",
78+
"dynamic": false,
79+
"properties": {
80+
"key": {
81+
"type": "keyword",
82+
"ignore_above": 256
83+
},
84+
"value": {
85+
"type": "keyword",
86+
"ignore_above": 256
87+
},
88+
"type": {
89+
"type": "keyword",
90+
"ignore_above": 256
91+
}
92+
}
93+
}
94+
}
95+
},
96+
"process": {
97+
"properties": {
98+
"serviceName": {
99+
"type": "keyword",
100+
"ignore_above": 256
101+
},
102+
"tag": {
103+
"type": "object"
104+
},
105+
"tags": {
106+
"type": "nested",
107+
"dynamic": false,
108+
"properties": {
109+
"key": {
110+
"type": "keyword",
111+
"ignore_above": 256
112+
},
113+
"value": {
114+
"type": "keyword",
115+
"ignore_above": 256
116+
},
117+
"type": {
118+
"type": "keyword",
119+
"ignore_above": 256
120+
}
121+
}
122+
}
123+
}
124+
},
125+
"references": {
126+
"type": "nested",
127+
"dynamic": false,
128+
"properties": {
129+
"refType": {
130+
"type": "keyword",
131+
"ignore_above": 256
132+
},
133+
"traceID": {
134+
"type": "keyword",
135+
"ignore_above": 256
136+
},
137+
"spanID": {
138+
"type": "keyword",
139+
"ignore_above": 256
140+
}
141+
}
142+
},
143+
"tag": {
144+
"type": "object"
145+
},
146+
"tags": {
147+
"type": "nested",
148+
"dynamic": false,
149+
"properties": {
150+
"key": {
151+
"type": "keyword",
152+
"ignore_above": 256
153+
},
154+
"value": {
155+
"type": "keyword",
156+
"ignore_above": 256
157+
},
158+
"type": {
159+
"type": "keyword",
160+
"ignore_above": 256
161+
}
162+
}
163+
}
164+
}
165+
}
166+
}
167+
}

‎internal/storage/v1/elasticsearch/mappings/jaeger-dependencies-ds-8.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"priority": {{ .Priority }},
3+
"index_patterns": "{{ .IndexPrefix }}jaeger-dependencies-ds",
4+
"data_stream": {},
5+
"template": {
6+
"settings": {
7+
"index.number_of_shards": {{ .Shards }},
8+
"index.number_of_replicas": {{ .Replicas }},
9+
"index.mapping.nested_fields.limit": 50,
10+
"index.requests.cache.enable": true,
11+
"index.default_pipeline": "jaeger-dependencies-ds-timestamp",
12+
"index.lifecycle.name": "{{ .ILMPolicyName }}"
13+
},
14+
"mappings": {
15+
"properties": {
16+
"@timestamp": {
17+
"type": "date",
18+
"format": "epoch_millis"
19+
}
20+
}
21+
}
22+
}
23+
}

‎internal/storage/v1/elasticsearch/mappings/jaeger-sampling-ds-8.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"priority": {{ .Priority }},
3+
"index_patterns": "{{ .IndexPrefix }}jaeger-sampling-ds",
4+
"data_stream": {},
5+
"template": {
6+
"settings": {
7+
"index.number_of_shards": {{ .Shards }},
8+
"index.number_of_replicas": {{ .Replicas }},
9+
"index.mapping.nested_fields.limit": 50,
10+
"index.requests.cache.enable": false,
11+
"index.default_pipeline": "jaeger-sampling-ds-timestamp",
12+
"index.lifecycle.name": "{{ .ILMPolicyName }}"
13+
},
14+
"mappings": {
15+
"properties": {
16+
"@timestamp": {
17+
"type": "date",
18+
"format": "epoch_millis"
19+
}
20+
}
21+
}
22+
}
23+
}

‎internal/storage/v1/elasticsearch/mappings/jaeger-service-ds-8.json‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
{
2+
"priority": {{ .Priority }},
3+
"index_patterns": "{{ .IndexPrefix }}jaeger-service-ds",
4+
"data_stream": {},
5+
"template": {
6+
"settings": {
7+
"index.number_of_shards": {{ .Shards }},
8+
"index.number_of_replicas": {{ .Replicas }},
9+
"index.mapping.nested_fields.limit": 50,
10+
"index.requests.cache.enable": true,
11+
"index.default_pipeline": "jaeger-service-ds-timestamp",
12+
"index.lifecycle.name": "{{ .ILMPolicyName }}"
13+
},
14+
"mappings": {
15+
"dynamic_templates": [
16+
{
17+
"span_tags_map": {
18+
"mapping": {
19+
"type": "keyword",
20+
"ignore_above": 256
21+
},
22+
"path_match": "tag.*"
23+
}
24+
},
25+
{
26+
"process_tags_map": {
27+
"mapping": {
28+
"type": "keyword",
29+
"ignore_above": 256
30+
},
31+
"path_match": "process.tag.*"
32+
}
33+
}
34+
],
35+
"properties": {
36+
"@timestamp": {
37+
"type": "date",
38+
"format": "epoch_millis"
39+
},
40+
"serviceName": {
41+
"type": "keyword",
42+
"ignore_above": 256
43+
},
44+
"operationName": {
45+
"type": "keyword",
46+
"ignore_above": 256
47+
}
48+
}
49+
}
50+
}
51+
}

0 commit comments

Comments
 (0)