Skip to content

Commit 4afacd0

Browse files
eyalkoreneyalkoren
committed
Applying review suggestions
1 parent 4773cf6 commit 4afacd0

File tree

6 files changed

+92
-38
lines changed

6 files changed

+92
-38
lines changed

‎docs/changelog/125699.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
pr: 125699
2-
summary: Adding `EcsNamespacingProcessor`
2+
summary: Adding `EcsNamespaceProcessor`
33
area: Ingest Node
44
type: feature
55
issues: []

‎modules/ingest-ecs/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
1212

1313
esplugin {
1414
description = 'Ingest processor that applies ECS namespacing'
15-
classname ='org.elasticsearch.ingest.ecs.EcsNamespacingPlugin'
15+
classname ='org.elasticsearch.ingest.ecs.EcsNamespacePlugin'
1616
}
1717

1818
restResources {

‎modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingPlugin.java‎ renamed to ‎modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacePlugin.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@
1515

1616
import java.util.Map;
1717

18-
public class EcsNamespacingPlugin extends Plugin implements IngestPlugin {
18+
public class EcsNamespacePlugin extends Plugin implements IngestPlugin {
1919

2020
@Override
2121
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
22-
return Map.of(EcsNamespacingProcessor.TYPE, new EcsNamespacingProcessor.Factory());
22+
return Map.of(EcsNamespaceProcessor.TYPE, new EcsNamespaceProcessor.Factory());
2323
}
2424
}

‎modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java‎ renamed to ‎modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java‎

Lines changed: 65 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,28 @@
2121
import java.util.Map;
2222
import java.util.Set;
2323

24-
public class EcsNamespacingProcessor extends AbstractProcessor {
24+
/**
25+
* This processor is responsible for transforming non-OpenTelemetry-compliant documents into a namespaced flavor of ECS
26+
* that makes them compatible with OpenTelemetry.
27+
* It DOES NOT translate the entire ECS schema into OpenTelemetry semantic conventions.
28+
*
29+
* <p>More specifically, this processor performs the following operations:
30+
* <ul>
31+
* <li>Renames specific ECS fields to their corresponding OpenTelemetry-compatible counterparts.</li>
32+
* <li>Moves fields to the "attributes" and "resource.attributes" namespaces.</li>
33+
* <li>Flattens the "attributes" and "resource.attributes" maps.</li>
34+
* </ul>
35+
*
36+
* <p>If a document is identified as OpenTelemetry-compatible, no transformation is performed.
37+
* @see org.elasticsearch.ingest.AbstractProcessor
38+
*/
39+
public class EcsNamespaceProcessor extends AbstractProcessor {
2540

26-
public static final String TYPE = "ecs_namespacing";
41+
public static final String TYPE = "ecs_namespace";
2742

43+
/**
44+
* Mapping of ECS field names to their corresponding OpenTelemetry-compatible counterparts.
45+
*/
2846
private static final Map<String, String> RENAME_KEYS = Map.of(
2947
"span.id",
3048
"span_id",
@@ -36,11 +54,18 @@ public class EcsNamespacingProcessor extends AbstractProcessor {
3654
"trace_id"
3755
);
3856

57+
/**
58+
* A close-set of keys that should be kept at the top level of the processed document after applying the namespacing.
59+
* In essence, these are the fields that should not be moved to the "attributes" or "resource.attributes" namespaces.
60+
* Besides the @timestamp field, this set obviously contains the attributes and the resource fields, as well as the
61+
* OpenTelemetry-compatible fields that are renamed by the processor.
62+
*/
3963
private static final Set<String> KEEP_KEYS;
4064
static {
4165
Set<String> keepKeys = new HashSet<>(Set.of("@timestamp", "attributes", "resource"));
4266
Set<String> renamedTopLevelFields = new HashSet<>();
4367
for (String value : RENAME_KEYS.values()) {
68+
// if the renamed field is nested, we only need to know the top level field
4469
int dotIndex = value.indexOf('.');
4570
if (dotIndex != -1) {
4671
renamedTopLevelFields.add(value.substring(0, dotIndex));
@@ -63,7 +88,7 @@ public class EcsNamespacingProcessor extends AbstractProcessor {
6388
private static final String TEXT_KEY = "text";
6489
private static final String STRUCTURED_KEY = "structured";
6590

66-
EcsNamespacingProcessor(String tag, String description) {
91+
EcsNamespaceProcessor(String tag, String description) {
6792
super(tag, description);
6893
}
6994

@@ -85,7 +110,9 @@ public IngestDocument execute(IngestDocument document) {
85110

86111
Map<String, Object> newAttributes = new HashMap<>();
87112
// The keep keys indicate the fields that should be kept at the top level later on when applying the namespacing.
88-
// However, at this point we need to move their original values to the new attributes namespace, except for the @timestamp field.
113+
// However, at this point we need to move their original values (if they exist) to the one of the new attributes namespaces, except
114+
// for the @timestamp field. The assumption is that at this point the document is not OTel compliant, so even if a valid top
115+
// level field is found, we assume that it does not bear the OTel semantics.
89116
for (String keepKey : KEEP_KEYS) {
90117
if (keepKey.equals("@timestamp")) {
91118
continue;
@@ -126,7 +153,24 @@ public IngestDocument execute(IngestDocument document) {
126153
return document;
127154
}
128155

129-
boolean isOTelDocument(Map<String, Object> source) {
156+
/**
157+
* Checks if the given document is OpenTelemetry-compliant.
158+
*
159+
* <p>A document is considered OpenTelemetry-compliant if it meets the following criteria:
160+
* <ul>
161+
* <li>The "resource" field is present and is a map
162+
* <li>The resource field either doesn't contain an "attributes" field, or the "attributes" field is a map.</li>
163+
* <li>The "scope" field is either absent or a map.</li>
164+
* <li>The "attributes" field is either absent or a map.</li>
165+
* <li>The "body" field is either absent or a map.</li>
166+
* <li>If exists, the "body" either doesn't contain a "text" field, or the "text" field is a string.</li>
167+
* <li>If exists, the "body" either doesn't contain a "structured" field, or the "structured" field is not a string.</li>
168+
* </ul>
169+
*
170+
* @param source the document to check
171+
* @return {@code true} if the document is OpenTelemetry-compliant, {@code false} otherwise
172+
*/
173+
static boolean isOTelDocument(Map<String, Object> source) {
130174
Object resource = source.get(RESOURCE_KEY);
131175
if (resource instanceof Map<?, ?> resourceMap) {
132176
Object resourceAttributes = resourceMap.get(ATTRIBUTES_KEY);
@@ -163,7 +207,21 @@ boolean isOTelDocument(Map<String, Object> source) {
163207
return true;
164208
}
165209

166-
private void renameSpecialKeys(IngestDocument document) {
210+
/**
211+
* Renames specific ECS keys in the given document to their OpenTelemetry-compatible counterparts, based on the {@code RENAME_KEYS} map.
212+
*
213+
* <p>This method performs the following operations:
214+
* <ul>
215+
* <li>For each key in the {@code RENAME_KEYS} map, it checks if a corresponding field exists in the document. If first looks for the
216+
* field assuming dot notation for nested fields. If the field is not found, it looks for a top level field with a dotted name.</li>
217+
* <li>If the field exists, it removes if from the document and adds a new field with the corresponding name from the {@code
218+
* RENAME_KEYS} map and the same value.</li>
219+
* <li>If the key is nested (contains dots), it recursively removes empty parent fields after renaming.</li>
220+
* </ul>
221+
*
222+
* @param document the document to process
223+
*/
224+
static void renameSpecialKeys(IngestDocument document) {
167225
RENAME_KEYS.forEach((nonOtelName, otelName) -> {
168226
// first look assuming dot notation for nested fields
169227
Object value = document.getFieldValue(nonOtelName, Object.class, true);
@@ -211,7 +269,7 @@ public Processor create(
211269
Map<String, Object> config,
212270
ProjectId projectId
213271
) throws Exception {
214-
return new EcsNamespacingProcessor(tag, description);
272+
return new EcsNamespaceProcessor(tag, description);
215273
}
216274
}
217275
}

‎modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java‎ renamed to ‎modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java‎

Lines changed: 16 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,35 +17,35 @@
1717
import java.util.Map;
1818

1919
@SuppressWarnings("unchecked")
20-
public class EcsNamespacingProcessorTests extends ESTestCase {
20+
public class EcsNamespaceProcessorTests extends ESTestCase {
2121

22-
private final EcsNamespacingProcessor processor = new EcsNamespacingProcessor("test", "test processor");
22+
private final EcsNamespaceProcessor processor = new EcsNamespaceProcessor("test", "test processor");
2323

2424
public void testIsOTelDocument_validMinimalOTelDocument() {
2525
Map<String, Object> document = new HashMap<>();
2626
document.put("resource", new HashMap<>());
27-
assertTrue(processor.isOTelDocument(document));
27+
assertTrue(EcsNamespaceProcessor.isOTelDocument(document));
2828
}
2929

3030
public void testIsOTelDocument_validOTelDocumentWithScopeAndAttributes() {
3131
Map<String, Object> document = new HashMap<>();
3232
document.put("attributes", new HashMap<>());
3333
document.put("resource", new HashMap<>());
3434
document.put("scope", new HashMap<>());
35-
assertTrue(processor.isOTelDocument(document));
35+
assertTrue(EcsNamespaceProcessor.isOTelDocument(document));
3636
}
3737

3838
public void testIsOTelDocument_missingResource() {
3939
Map<String, Object> document = new HashMap<>();
4040
document.put("scope", new HashMap<>());
41-
assertFalse(processor.isOTelDocument(document));
41+
assertFalse(EcsNamespaceProcessor.isOTelDocument(document));
4242
}
4343

4444
public void testIsOTelDocument_resourceNotMap() {
4545
Map<String, Object> document = new HashMap<>();
4646
document.put("resource", "not a map");
4747
document.put("scope", new HashMap<>());
48-
assertFalse(processor.isOTelDocument(document));
48+
assertFalse(EcsNamespaceProcessor.isOTelDocument(document));
4949
}
5050

5151
public void testIsOTelDocument_invalidResourceAttributes() {
@@ -54,30 +54,30 @@ public void testIsOTelDocument_invalidResourceAttributes() {
5454
Map<String, Object> document = new HashMap<>();
5555
document.put("resource", resource);
5656
document.put("scope", new HashMap<>());
57-
assertFalse(processor.isOTelDocument(document));
57+
assertFalse(EcsNamespaceProcessor.isOTelDocument(document));
5858
}
5959

6060
public void testIsOTelDocument_scopeNotMap() {
6161
Map<String, Object> document = new HashMap<>();
6262
document.put("resource", new HashMap<>());
6363
document.put("scope", "not a map");
64-
assertFalse(processor.isOTelDocument(document));
64+
assertFalse(EcsNamespaceProcessor.isOTelDocument(document));
6565
}
6666

6767
public void testIsOTelDocument_invalidAttributes() {
6868
Map<String, Object> document = new HashMap<>();
6969
document.put("resource", new HashMap<>());
7070
document.put("scope", new HashMap<>());
7171
document.put("attributes", "not a map");
72-
assertFalse(processor.isOTelDocument(document));
72+
assertFalse(EcsNamespaceProcessor.isOTelDocument(document));
7373
}
7474

7575
public void testIsOTelDocument_invalidBody() {
7676
Map<String, Object> document = new HashMap<>();
7777
document.put("resource", new HashMap<>());
7878
document.put("scope", new HashMap<>());
7979
document.put("body", "not a map");
80-
assertFalse(processor.isOTelDocument(document));
80+
assertFalse(EcsNamespaceProcessor.isOTelDocument(document));
8181
}
8282

8383
public void testIsOTelDocument_invalidBodyText() {
@@ -87,7 +87,7 @@ public void testIsOTelDocument_invalidBodyText() {
8787
document.put("resource", new HashMap<>());
8888
document.put("scope", new HashMap<>());
8989
document.put("body", body);
90-
assertFalse(processor.isOTelDocument(document));
90+
assertFalse(EcsNamespaceProcessor.isOTelDocument(document));
9191
}
9292

9393
public void testIsOTelDocument_invalidBodyStructured() {
@@ -97,7 +97,7 @@ public void testIsOTelDocument_invalidBodyStructured() {
9797
document.put("resource", new HashMap<>());
9898
document.put("scope", new HashMap<>());
9999
document.put("body", body);
100-
assertFalse(processor.isOTelDocument(document));
100+
assertFalse(EcsNamespaceProcessor.isOTelDocument(document));
101101
}
102102

103103
public void testIsOTelDocument_validBody() {
@@ -108,7 +108,7 @@ public void testIsOTelDocument_validBody() {
108108
document.put("resource", new HashMap<>());
109109
document.put("scope", new HashMap<>());
110110
document.put("body", body);
111-
assertTrue(processor.isOTelDocument(document));
111+
assertTrue(EcsNamespaceProcessor.isOTelDocument(document));
112112
}
113113

114114
public void testExecute_validOTelDocument() {
@@ -210,7 +210,7 @@ public void testRenameSpecialKeys_nestedForm() {
210210
document.put("trace", trace);
211211
IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document);
212212

213-
processor.execute(ingestDocument);
213+
EcsNamespaceProcessor.renameSpecialKeys(ingestDocument);
214214

215215
Map<String, Object> source = ingestDocument.getSource();
216216
assertEquals("spanIdValue", source.get("span_id"));
@@ -219,8 +219,6 @@ public void testRenameSpecialKeys_nestedForm() {
219219
assertFalse(source.containsKey("log"));
220220
assertEquals("traceIdValue", source.get("trace_id"));
221221
assertFalse(source.containsKey("trace"));
222-
assertTrue(source.containsKey("attributes"));
223-
assertTrue(((Map<String, Object>) source.get("attributes")).isEmpty());
224222
}
225223

226224
public void testRenameSpecialKeys_topLevelDottedField() {
@@ -231,7 +229,7 @@ public void testRenameSpecialKeys_topLevelDottedField() {
231229
document.put("message", "this is a message");
232230
IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document);
233231

234-
processor.execute(ingestDocument);
232+
EcsNamespaceProcessor.renameSpecialKeys(ingestDocument);
235233

236234
Map<String, Object> source = ingestDocument.getSource();
237235
assertEquals("spanIdValue", source.get("span_id"));
@@ -240,8 +238,6 @@ public void testRenameSpecialKeys_topLevelDottedField() {
240238
Object body = source.get("body");
241239
assertTrue(body instanceof Map);
242240
assertEquals("this is a message", ((Map<String, Object>) body).get("text"));
243-
assertTrue(source.containsKey("attributes"));
244-
assertTrue(((Map<String, Object>) source.get("attributes")).isEmpty());
245241
assertFalse(document.containsKey("span.id"));
246242
assertFalse(document.containsKey("log.level"));
247243
assertFalse(document.containsKey("trace.id"));
@@ -256,7 +252,7 @@ public void testRenameSpecialKeys_mixedForm() {
256252
document.put("span.id", "topLevelSpanIdValue");
257253
IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document);
258254

259-
processor.execute(ingestDocument);
255+
EcsNamespaceProcessor.renameSpecialKeys(ingestDocument);
260256

261257
Map<String, Object> source = ingestDocument.getSource();
262258
// nested form should take precedence

‎modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml‎ renamed to ‎modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespace.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@
22
setup:
33
- do:
44
ingest.put_pipeline:
5-
id: "ecs_namespacing_pipeline"
5+
id: "ecs_namespace_pipeline"
66
body:
77
processors:
8-
- ecs_namespacing: {}
8+
- ecs_namespace: {}
99

1010
---
1111
teardown:
1212
- do:
1313
ingest.delete_pipeline:
14-
id: "ecs_namespacing_pipeline"
14+
id: "ecs_namespace_pipeline"
1515
ignore: 404
1616

1717
---
@@ -20,7 +20,7 @@ teardown:
2020
index:
2121
index: ecs_namespacing_test
2222
id: "nested_and_flat_attributes"
23-
pipeline: "ecs_namespacing_pipeline"
23+
pipeline: "ecs_namespace_pipeline"
2424
body: {
2525
"agent.name": "agentNameValue",
2626
"agent": {
@@ -111,7 +111,7 @@ teardown:
111111
index:
112112
index: ecs_namespacing_test
113113
id: "rename_special_keys"
114-
pipeline: "ecs_namespacing_pipeline"
114+
pipeline: "ecs_namespace_pipeline"
115115
body: {
116116
"span": {
117117
"id": "nestedSpanIdValue"
@@ -146,7 +146,7 @@ teardown:
146146
index:
147147
index: ecs_namespacing_test
148148
id: "valid_otel_document"
149-
pipeline: "ecs_namespacing_pipeline"
149+
pipeline: "ecs_namespace_pipeline"
150150
body: {
151151
"resource": {
152152
"attributes": {
@@ -189,7 +189,7 @@ teardown:
189189
index:
190190
index: ecs_namespacing_test
191191
id: "invalid_body_field"
192-
pipeline: "ecs_namespacing_pipeline"
192+
pipeline: "ecs_namespace_pipeline"
193193
body: {
194194
"resource": {},
195195
"scope": {

0 commit comments

Comments
 (0)