From 30b0ed34e1029a297ccf6734c4694ef4ac19efbd Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Wed, 26 Mar 2025 18:55:01 +0200 Subject: [PATCH 01/45] Adding EcsNamespacingProcessor --- modules/ingest-ecs/build.gradle | 22 + .../ingest/ecs/EcsNamespacingProcessor.java | 203 +++++++++ .../ecs/EcsNamespacingProcessorTests.java | 408 ++++++++++++++++++ 3 files changed, 633 insertions(+) create mode 100644 modules/ingest-ecs/build.gradle create mode 100644 modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java create mode 100644 modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java diff --git a/modules/ingest-ecs/build.gradle b/modules/ingest-ecs/build.gradle new file mode 100644 index 0000000000000..c35c91b1d6a3d --- /dev/null +++ b/modules/ingest-ecs/build.gradle @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' + +esplugin { + description = 'Ingest processor that applies ECS namespacing' + classname ='org.elasticsearch.ingest.ecs.EcsNamespacingPlugin' +} + +restResources { + restApi { + include '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest' + } +} diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java new file mode 100644 index 0000000000000..b231679772cf6 --- /dev/null +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.ecs; + +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class EcsNamespacingProcessor extends AbstractProcessor { + + public static final String TYPE = "ecs_namespacing"; + + private static final Set KEEP_KEYS = Set.of( + "@timestamp", + "observed_timestamp", + "trace_id", + "span_id", + "severity_text", + "body", + "severity_number", + "event_name", + "attributes", + "resource", + "dropped_attributes_count", + "scope" + ); + + private static final Map RENAME_KEYS = Map.of( + "span.id", + "span_id", + "message", + "body.text", + "log.level", + "severity_text", + "trace.id", + "trace_id" + ); + + private static final String AGENT_PREFIX = "agent."; + private static final String CLOUD_PREFIX = "cloud."; + private static final String HOST_PREFIX = "host."; + + private static final String ATTRIBUTES_KEY = "attributes"; + private static final String RESOURCE_KEY = "resource"; + private static final String SCOPE_KEY = "scope"; + private static final String BODY_KEY = "body"; + private static final String TEXT_KEY = "text"; + private static final String STRUCTURED_KEY = "structured"; + + EcsNamespacingProcessor(String tag, String description) { + super(tag, description); + } + + @Override + public String getType() { + return TYPE; + } + + @Override + public IngestDocument execute(IngestDocument document) { + Map source = document.getSource(); + + boolean isOTel = isOTelDocument(source); + if (isOTel) { + return document; + } + + // non-OTel document + + Map newAttributes = new HashMap<>(); + Object oldAttributes = source.remove(ATTRIBUTES_KEY); + if (oldAttributes != null) { + newAttributes.put(ATTRIBUTES_KEY, oldAttributes); + } + source.put(ATTRIBUTES_KEY, newAttributes); + + Map newResource = new HashMap<>(); + Map newResourceAttributes = new HashMap<>(); + newResource.put(ATTRIBUTES_KEY, newResourceAttributes); + Object oldResource = source.remove(RESOURCE_KEY); + if (oldResource != null) { + newAttributes.put(RESOURCE_KEY, oldResource); + } + source.put(RESOURCE_KEY, newResource); + + renameSpecialKeys(document); + + // Iterate through all top level keys and move them to the appropriate namespace + for (String key : Sets.newHashSet(source.keySet())) { + if (KEEP_KEYS.contains(key)) { + continue; + } + if (shouldMoveToResourceAttributes(key)) { + Object value = source.remove(key); + newResourceAttributes.put(key, value); + } else { + Object value = source.remove(key); + newAttributes.put(key, value); + } + } + + // Flatten attributes + source.replace(ATTRIBUTES_KEY, Maps.flatten(newAttributes, false, false)); + newResource.replace(ATTRIBUTES_KEY, Maps.flatten(newResourceAttributes, false, false)); + + return document; + } + + @SuppressWarnings("unchecked") + boolean isOTelDocument(Map source) { + Object resource = source.get(RESOURCE_KEY); + if (resource instanceof Map == false) { + return false; + } else { + Object resourceAttributes = ((Map) resource).get(ATTRIBUTES_KEY); + if (resourceAttributes != null && (resourceAttributes instanceof Map) == false) { + return false; + } + } + + Object scope = source.get(SCOPE_KEY); + if (scope instanceof Map == false) { + return false; + } + + Object attributes = source.get(ATTRIBUTES_KEY); + if (attributes != null && attributes instanceof Map == false) { + return false; + } + + Object body = source.get(BODY_KEY); + if (body != null) { + if (body instanceof Map == false) { + return false; + } + Object bodyText = ((Map) body).get(TEXT_KEY); + if (bodyText != null && (bodyText instanceof String) == false) { + return false; + } + Object bodyStructured = ((Map) body).get(STRUCTURED_KEY); + return (bodyStructured instanceof String) == false; + } + return true; + } + + private void renameSpecialKeys(IngestDocument document) { + RENAME_KEYS.forEach((nonOtelName, otelName) -> { + // first look assuming dot notation + Object value = document.getFieldValue(nonOtelName, Object.class, true); + if (value != null) { + document.removeField(nonOtelName); + // remove the parent field if it is empty + int lastDot = nonOtelName.lastIndexOf('.'); + if (lastDot > 0) { + String parent = nonOtelName.substring(0, lastDot); + document.removeField(parent, true); + } + } else if (nonOtelName.contains(".")) { + // look for dotted field names + value = document.getSource().remove(nonOtelName); + } + if (value != null) { + document.setFieldValue(otelName, value); + } + }); + } + + private boolean shouldMoveToResourceAttributes(String key) { + return key.startsWith(AGENT_PREFIX) + || key.equals("agent") + || key.startsWith(CLOUD_PREFIX) + || key.equals("cloud") + || key.startsWith(HOST_PREFIX) + || key.equals("host"); + } + + public static final class Factory implements Processor.Factory { + @Override + public Processor create( + Map processorFactories, + String tag, + String description, + Map config, + ProjectId projectId + ) throws Exception { + return new EcsNamespacingProcessor(tag, description); + } + } +} diff --git a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java new file mode 100644 index 0000000000000..eaf322b15ea67 --- /dev/null +++ b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java @@ -0,0 +1,408 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.ecs; + +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@SuppressWarnings("unchecked") +public class EcsNamespacingProcessorTests extends ESTestCase { + + private final EcsNamespacingProcessor processor = new EcsNamespacingProcessor("test", "test processor"); + + public void testIsOTelDocument_validMinimalOTelDocument() { + Map document = new HashMap<>(); + document.put("resource", new HashMap<>()); + document.put("scope", new HashMap<>()); + assertTrue(processor.isOTelDocument(document)); + } + + public void testIsOTelDocument_validOTelDocumentWithAttributes() { + Map document = new HashMap<>(); + document.put("attributes", new HashMap<>()); + document.put("resource", new HashMap<>()); + document.put("scope", new HashMap<>()); + assertTrue(processor.isOTelDocument(document)); + } + + public void testIsOTelDocument_missingResource() { + Map document = new HashMap<>(); + document.put("scope", new HashMap<>()); + assertFalse(processor.isOTelDocument(document)); + } + + public void testIsOTelDocument_missingScope() { + Map document = new HashMap<>(); + document.put("resource", new HashMap<>()); + assertFalse(processor.isOTelDocument(document)); + } + + public void testIsOTelDocument_resourceNotMap() { + Map document = new HashMap<>(); + document.put("resource", "not a map"); + document.put("scope", new HashMap<>()); + assertFalse(processor.isOTelDocument(document)); + } + + public void testIsOTelDocument_invalidResourceAttributes() { + Map resource = new HashMap<>(); + resource.put("attributes", "not a map"); + Map document = new HashMap<>(); + document.put("resource", resource); + document.put("scope", new HashMap<>()); + assertFalse(processor.isOTelDocument(document)); + } + + public void testIsOTelDocument_scopeNotMap() { + Map document = new HashMap<>(); + document.put("resource", new HashMap<>()); + document.put("scope", "not a map"); + assertFalse(processor.isOTelDocument(document)); + } + + public void testIsOTelDocument_invalidAttributes() { + Map document = new HashMap<>(); + document.put("resource", new HashMap<>()); + document.put("scope", new HashMap<>()); + document.put("attributes", "not a map"); + assertFalse(processor.isOTelDocument(document)); + } + + public void testIsOTelDocument_invalidBody() { + Map document = new HashMap<>(); + document.put("resource", new HashMap<>()); + document.put("scope", new HashMap<>()); + document.put("body", "not a map"); + assertFalse(processor.isOTelDocument(document)); + } + + public void testIsOTelDocument_invalidBodyText() { + Map body = new HashMap<>(); + body.put("text", 123); + Map document = new HashMap<>(); + document.put("resource", new HashMap<>()); + document.put("scope", new HashMap<>()); + document.put("body", body); + assertFalse(processor.isOTelDocument(document)); + } + + public void testIsOTelDocument_invalidBodyStructured() { + Map body = new HashMap<>(); + body.put("structured", "a string"); + Map document = new HashMap<>(); + document.put("resource", new HashMap<>()); + document.put("scope", new HashMap<>()); + document.put("body", body); + assertFalse(processor.isOTelDocument(document)); + } + + public void testIsOTelDocument_validBody() { + Map body = new HashMap<>(); + body.put("text", "a string"); + body.put("structured", new HashMap<>()); + Map document = new HashMap<>(); + document.put("resource", new HashMap<>()); + document.put("scope", new HashMap<>()); + document.put("body", body); + assertTrue(processor.isOTelDocument(document)); + } + + public void testExecute_validOTelDocument() { + Map document = new HashMap<>(); + document.put("resource", new HashMap<>()); + document.put("scope", new HashMap<>()); + Map body = new HashMap<>(); + body.put("text", "a string"); + body.put("structured", new HashMap<>()); + document.put("body", body); + document.put("key1", "value1"); + Map before = new HashMap<>(document); + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + IngestDocument result = processor.execute(ingestDocument); + assertEquals(before, result.getSource()); + } + + public void testExecute_nonOTelDocument() { + Map document = new HashMap<>(); + document.put("key1", "value1"); + document.put("key2", "value2"); + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + + IngestDocument result = processor.execute(ingestDocument); + + Map source = result.getSource(); + assertTrue(source.containsKey("attributes")); + assertTrue(source.containsKey("resource")); + + Map attributes = (Map) source.get("attributes"); + assertEquals("value1", attributes.get("key1")); + assertEquals("value2", attributes.get("key2")); + assertFalse(document.containsKey("key1")); + assertFalse(document.containsKey("key2")); + + Map resource = (Map) source.get("resource"); + assertTrue(resource.containsKey("attributes")); + assertTrue(((Map) resource.get("attributes")).isEmpty()); + } + + public void testExecute_nonOTelDocument_withExistingAttributes() { + Map document = new HashMap<>(); + Map existingAttributes = new HashMap<>(); + existingAttributes.put("existingKey", "existingValue"); + document.put("attributes", existingAttributes); + document.put("key1", "value1"); + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + + IngestDocument result = processor.execute(ingestDocument); + + Map source = result.getSource(); + assertTrue(source.containsKey("attributes")); + assertTrue(source.containsKey("resource")); + + Map attributes = (Map) source.get("attributes"); + assertEquals("existingValue", attributes.get("attributes.existingKey")); + assertEquals("value1", attributes.get("key1")); + + Map resource = (Map) source.get("resource"); + assertTrue(resource.containsKey("attributes")); + assertTrue(((Map) resource.get("attributes")).isEmpty()); + } + + public void testExecute_nonOTelDocument_withExistingResource() { + Map document = new HashMap<>(); + Map existingResource = new HashMap<>(); + existingResource.put("existingKey", "existingValue"); + document.put("resource", existingResource); + document.put("key1", "value1"); + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + + IngestDocument result = processor.execute(ingestDocument); + + Map source = result.getSource(); + assertTrue(source.containsKey("attributes")); + assertTrue(source.containsKey("resource")); + + Map attributes = (Map) source.get("attributes"); + assertEquals("value1", attributes.get("key1")); + assertEquals("existingValue", attributes.get("resource.existingKey")); + + Map resource = (Map) source.get("resource"); + assertTrue(resource.containsKey("attributes")); + assertTrue(((Map) resource.get("attributes")).isEmpty()); + } + + public void testRenameSpecialKeys_nestedForm() { + Map document = new HashMap<>(); + Map span = new HashMap<>(); + span.put("id", "spanIdValue"); + document.put("span", span); + Map log = new HashMap<>(); + log.put("level", "logLevelValue"); + document.put("log", log); + Map trace = new HashMap<>(); + trace.put("id", "traceIdValue"); + document.put("trace", trace); + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + + processor.execute(ingestDocument); + + Map source = ingestDocument.getSource(); + assertEquals("spanIdValue", source.get("span_id")); + assertFalse(source.containsKey("span")); + assertEquals("logLevelValue", source.get("severity_text")); + assertFalse(source.containsKey("log")); + assertEquals("traceIdValue", source.get("trace_id")); + assertFalse(source.containsKey("trace")); + assertTrue(source.containsKey("attributes")); + assertTrue(((Map) source.get("attributes")).isEmpty()); + } + + public void testRenameSpecialKeys_topLevelDottedField() { + Map document = new HashMap<>(); + document.put("span.id", "spanIdValue"); + document.put("log.level", "logLevelValue"); + document.put("trace.id", "traceIdValue"); + document.put("message", "this is a message"); + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + + processor.execute(ingestDocument); + + Map source = ingestDocument.getSource(); + assertEquals("spanIdValue", source.get("span_id")); + assertEquals("logLevelValue", source.get("severity_text")); + assertEquals("traceIdValue", source.get("trace_id")); + Object body = source.get("body"); + assertTrue(body instanceof Map); + assertEquals("this is a message", ((Map) body).get("text")); + assertTrue(source.containsKey("attributes")); + assertTrue(((Map) source.get("attributes")).isEmpty()); + assertFalse(document.containsKey("span.id")); + assertFalse(document.containsKey("log.level")); + assertFalse(document.containsKey("trace.id")); + assertFalse(document.containsKey("message")); + } + + public void testRenameSpecialKeys_mixedForm() { + Map document = new HashMap<>(); + Map span = new HashMap<>(); + span.put("id", "nestedSpanIdValue"); + document.put("span", span); + document.put("span.id", "topLevelSpanIdValue"); + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + + processor.execute(ingestDocument); + + Map source = ingestDocument.getSource(); + // nested form should take precedence + assertEquals("nestedSpanIdValue", source.get("span_id")); + } + + public void testExecute_moveToAttributeMaps() { + Map document = new HashMap<>(); + document.put("agent.name", "agentNameValue"); + Map agent = new HashMap<>(); + agent.put("type", "agentTypeValue"); + document.put("agent", agent); + document.put("cloud.provider", "cloudProviderValue"); + Map cloud = new HashMap<>(); + cloud.put("type", "cloudTypeValue"); + document.put("cloud", cloud); + document.put("host.name", "hostNameValue"); + Map host = new HashMap<>(); + host.put("type", "hostTypeValue"); + document.put("host", host); + document.put("service.name", "serviceNameValue"); + Map service = new HashMap<>(); + service.put("type", "serviceTypeValue"); + document.put("service", service); + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + + IngestDocument result = processor.execute(ingestDocument); + + Map source = result.getSource(); + + // all attributes should be flattened + Map expectedResourceAttributes = Map.of( + "agent.name", + "agentNameValue", + "agent.type", + "agentTypeValue", + "cloud.provider", + "cloudProviderValue", + "cloud.type", + "cloudTypeValue", + "host.name", + "hostNameValue", + "host.type", + "hostTypeValue" + ); + + assertTrue(source.containsKey("resource")); + Map resource = (Map) source.get("resource"); + Map resourceAttributes = (Map) resource.get("attributes"); + assertEquals(expectedResourceAttributes, resourceAttributes); + assertNull(resourceAttributes.get("agent")); + assertNull(resourceAttributes.get("cloud")); + assertNull(resourceAttributes.get("host")); + assertFalse(document.containsKey("agent.name")); + assertFalse(document.containsKey("agent")); + assertFalse(document.containsKey("cloud.provider")); + assertFalse(document.containsKey("cloud")); + assertFalse(document.containsKey("host.name")); + assertFalse(document.containsKey("host")); + + Map expectedAttributes = Map.of("service.name", "serviceNameValue", "service.type", "serviceTypeValue"); + + assertTrue(source.containsKey("attributes")); + Map attributes = (Map) source.get("attributes"); + assertEquals(expectedAttributes, attributes); + assertNull(attributes.get("service")); + assertFalse(document.containsKey("service.name")); + assertFalse(document.containsKey("service")); + } + + public void testExecute_deepFlattening() { + Map document = new HashMap<>(); + Map nestedAgent = new HashMap<>(); + nestedAgent.put("name", "agentNameValue"); + Map deeperAgent = new HashMap<>(); + deeperAgent.put("type", "agentTypeValue"); + nestedAgent.put("details", deeperAgent); + document.put("agent", nestedAgent); + + Map nestedService = new HashMap<>(); + nestedService.put("name", "serviceNameValue"); + Map deeperService = new HashMap<>(); + deeperService.put("type", "serviceTypeValue"); + nestedService.put("details", deeperService); + document.put("service", nestedService); + + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + + IngestDocument result = processor.execute(ingestDocument); + + Map source = result.getSource(); + + Map expectedResourceAttributes = Map.of("agent.name", "agentNameValue", "agent.details.type", "agentTypeValue"); + + assertTrue(source.containsKey("resource")); + Map resource = (Map) source.get("resource"); + Map resourceAttributes = (Map) resource.get("attributes"); + assertEquals(expectedResourceAttributes, resourceAttributes); + assertNull(resource.get("agent")); + + Map expectedAttributes = Map.of("service.name", "serviceNameValue", "service.details.type", "serviceTypeValue"); + + assertTrue(source.containsKey("attributes")); + Map attributes = (Map) source.get("attributes"); + assertEquals(expectedAttributes, attributes); + assertNull(attributes.get("service")); + } + + public void testExecute_arraysNotFlattened() { + Map document = new HashMap<>(); + Map nestedAgent = new HashMap<>(); + nestedAgent.put("name", "agentNameValue"); + List agentArray = List.of("value1", "value2"); + nestedAgent.put("array", agentArray); + document.put("agent", nestedAgent); + + Map nestedService = new HashMap<>(); + nestedService.put("name", "serviceNameValue"); + List serviceArray = List.of("value1", "value2"); + nestedService.put("array", serviceArray); + document.put("service", nestedService); + + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + + IngestDocument result = processor.execute(ingestDocument); + + Map source = result.getSource(); + + Map expectedResourceAttributes = Map.of("agent.name", "agentNameValue", "agent.array", agentArray); + + assertTrue(source.containsKey("resource")); + Map resource = (Map) source.get("resource"); + Map resourceAttributes = (Map) resource.get("attributes"); + assertEquals(expectedResourceAttributes, resourceAttributes); + assertNull(resource.get("agent")); + + Map expectedAttributes = Map.of("service.name", "serviceNameValue", "service.array", serviceArray); + + assertTrue(source.containsKey("attributes")); + Map attributes = (Map) source.get("attributes"); + assertEquals(expectedAttributes, attributes); + assertNull(attributes.get("service")); + } +} From b96cc0c21e6158c02c1e23320a25d29abe462ee0 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 27 Mar 2025 07:49:32 +0200 Subject: [PATCH 02/45] Adding module-info --- .../ingest-ecs/src/main/java/module-info.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 modules/ingest-ecs/src/main/java/module-info.java diff --git a/modules/ingest-ecs/src/main/java/module-info.java b/modules/ingest-ecs/src/main/java/module-info.java new file mode 100644 index 0000000000000..08129bbb0d3be --- /dev/null +++ b/modules/ingest-ecs/src/main/java/module-info.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module org.elasticsearch.ingest.ecs { + requires org.elasticsearch.base; + requires org.elasticsearch.server; + requires org.elasticsearch.xcontent; + requires org.apache.logging.log4j; + requires org.elasticsearch.logging; +} From 2bd819f60e37151587628d82c4258379cc75e311 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 27 Mar 2025 11:08:27 +0200 Subject: [PATCH 03/45] Exposing and testing the processor --- .../ingest/ecs/EcsNamespacingPlugin.java | 24 +++++++ .../ecs/IngestEcsClientYamlTestSuiteIT.java | 38 ++++++++++++ .../test/ingest-ecs/10_ecs_namespacing.yml | 62 +++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingPlugin.java create mode 100644 modules/ingest-ecs/src/yamlRestTest/java/org/elasticsearch/ingest/ecs/IngestEcsClientYamlTestSuiteIT.java create mode 100644 modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingPlugin.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingPlugin.java new file mode 100644 index 0000000000000..683fd9bcb41c9 --- /dev/null +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingPlugin.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.ecs; + +import org.elasticsearch.ingest.Processor; +import org.elasticsearch.plugins.IngestPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.util.Map; + +public class EcsNamespacingPlugin extends Plugin implements IngestPlugin { + + @Override + public Map getProcessors(Processor.Parameters parameters) { + return Map.of(EcsNamespacingProcessor.TYPE, new EcsNamespacingProcessor.Factory()); + } +} diff --git a/modules/ingest-ecs/src/yamlRestTest/java/org/elasticsearch/ingest/ecs/IngestEcsClientYamlTestSuiteIT.java b/modules/ingest-ecs/src/yamlRestTest/java/org/elasticsearch/ingest/ecs/IngestEcsClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..c93f3ef875c22 --- /dev/null +++ b/modules/ingest-ecs/src/yamlRestTest/java/org/elasticsearch/ingest/ecs/IngestEcsClientYamlTestSuiteIT.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.ecs; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; + +public class IngestEcsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public IngestEcsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("ingest-ecs").build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml b/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml new file mode 100644 index 0000000000000..3f8f5d3b44242 --- /dev/null +++ b/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml @@ -0,0 +1,62 @@ +--- +setup: + - do: + ingest.put_pipeline: + id: "ecs_namespacing_pipeline" + body: + processors: + - ecs_namespacing: {} + +--- +teardown: + - do: + ingest.delete_pipeline: + id: "ecs_namespacing_pipeline" + ignore: 404 + +--- +"Test attributes namespacing": + - do: + index: + index: test + id: "1" + pipeline: "ecs_namespacing_pipeline" + body: { + "agent.name": "agentNameValue", + "agent": { + "type": "agentTypeValue" + }, + "cloud.provider": "cloudProviderValue", + "cloud": { + "type": "cloudTypeValue" + }, + "host.name": "hostNameValue", + "host": { + "type": "hostTypeValue" + }, + "service.name": "serviceNameValue", + "service": { + "type": "serviceTypeValue" + } + } + + - do: + get: + index: test + id: "1" + - match: { _source.resource.attributes.agent\.name: "agentNameValue" } + - match: { _source.resource.attributes.agent\.type: "agentTypeValue" } + - match: { _source.resource.attributes.cloud\.provider: "cloudProviderValue" } + - match: { _source.resource.attributes.cloud\.type: "cloudTypeValue" } + - match: { _source.resource.attributes.host\.name: "hostNameValue" } + - match: { _source.resource.attributes.host\.type: "hostTypeValue" } + - match: { _source.attributes.service\.name: "serviceNameValue" } + - match: { _source.attributes.service\.type: "serviceTypeValue" } + - match: { _source.agent.name: null } + - match: { _source.agent: null } + - match: { _source.cloud.provider: null } + - match: { _source.cloud: null } + - match: { _source.host.name: null } + - match: { _source.host: null } + - match: { _source.service.name: null } + - match: { _source.service: null } From 1c2a670168e4df4c68da63595c79f7b2ad19c681 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 27 Mar 2025 13:09:19 +0200 Subject: [PATCH 04/45] Add test and some algorithm fixes --- .../ingest/ecs/EcsNamespacingProcessor.java | 70 +++++++----- .../test/ingest-ecs/10_ecs_namespacing.yml | 108 +++++++++++++++++- 2 files changed, 146 insertions(+), 32 deletions(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java index b231679772cf6..86d745ecb21e2 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java @@ -17,6 +17,7 @@ import org.elasticsearch.ingest.Processor; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -24,21 +25,6 @@ public class EcsNamespacingProcessor extends AbstractProcessor { public static final String TYPE = "ecs_namespacing"; - private static final Set KEEP_KEYS = Set.of( - "@timestamp", - "observed_timestamp", - "trace_id", - "span_id", - "severity_text", - "body", - "severity_number", - "event_name", - "attributes", - "resource", - "dropped_attributes_count", - "scope" - ); - private static final Map RENAME_KEYS = Map.of( "span.id", "span_id", @@ -50,6 +36,22 @@ public class EcsNamespacingProcessor extends AbstractProcessor { "trace_id" ); + private static final Set KEEP_KEYS; + static { + Set keepKeys = new HashSet<>(Set.of("@timestamp", "attributes", "resource")); + Set renamedTopLevelFields = new HashSet<>(); + for (String value : RENAME_KEYS.values()) { + int dotIndex = value.indexOf('.'); + if (dotIndex != -1) { + renamedTopLevelFields.add(value.substring(0, dotIndex)); + } else { + renamedTopLevelFields.add(value); + } + } + keepKeys.addAll(renamedTopLevelFields); + KEEP_KEYS = Set.copyOf(keepKeys); + } + private static final String AGENT_PREFIX = "agent."; private static final String CLOUD_PREFIX = "cloud."; private static final String HOST_PREFIX = "host."; @@ -82,19 +84,23 @@ public IngestDocument execute(IngestDocument document) { // non-OTel document Map newAttributes = new HashMap<>(); - Object oldAttributes = source.remove(ATTRIBUTES_KEY); - if (oldAttributes != null) { - newAttributes.put(ATTRIBUTES_KEY, oldAttributes); + // The keep keys indicate the fields that should be kept at the top level later on when applying the namespacing. + // However, at this point we need to move their original values to the new attributes namespace, except for the @timestamp field. + for (String keepKey : KEEP_KEYS) { + if (keepKey.equals("@timestamp")) { + continue; + } + Object value = source.remove(keepKey); + if (value != null) { + newAttributes.put(keepKey, value); + } } - source.put(ATTRIBUTES_KEY, newAttributes); Map newResource = new HashMap<>(); Map newResourceAttributes = new HashMap<>(); newResource.put(ATTRIBUTES_KEY, newResourceAttributes); - Object oldResource = source.remove(RESOURCE_KEY); - if (oldResource != null) { - newAttributes.put(RESOURCE_KEY, oldResource); - } + + source.put(ATTRIBUTES_KEY, newAttributes); source.put(RESOURCE_KEY, newResource); renameSpecialKeys(document); @@ -159,15 +165,23 @@ boolean isOTelDocument(Map source) { private void renameSpecialKeys(IngestDocument document) { RENAME_KEYS.forEach((nonOtelName, otelName) -> { - // first look assuming dot notation + // first look assuming dot notation for nested fields Object value = document.getFieldValue(nonOtelName, Object.class, true); if (value != null) { document.removeField(nonOtelName); - // remove the parent field if it is empty + // recursively remove empty parent fields int lastDot = nonOtelName.lastIndexOf('.'); - if (lastDot > 0) { - String parent = nonOtelName.substring(0, lastDot); - document.removeField(parent, true); + while (lastDot > 0) { + String parentName = nonOtelName.substring(0, lastDot); + // parent should never be null and must be a map if we are here + @SuppressWarnings("unchecked") + Map parent = (Map) document.getFieldValue(parentName, Map.class); + if (parent.isEmpty()) { + document.removeField(parentName); + } else { + break; + } + lastDot = parentName.lastIndexOf('.'); } } else if (nonOtelName.contains(".")) { // look for dotted field names diff --git a/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml b/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml index 3f8f5d3b44242..6c09ba004d931 100644 --- a/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml +++ b/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml @@ -18,8 +18,8 @@ teardown: "Test attributes namespacing": - do: index: - index: test - id: "1" + index: ecs_namespacing_test + id: "nested_and_flat_attributes" pipeline: "ecs_namespacing_pipeline" body: { "agent.name": "agentNameValue", @@ -42,8 +42,8 @@ teardown: - do: get: - index: test - id: "1" + index: ecs_namespacing_test + id: "nested_and_flat_attributes" - match: { _source.resource.attributes.agent\.name: "agentNameValue" } - match: { _source.resource.attributes.agent\.type: "agentTypeValue" } - match: { _source.resource.attributes.cloud\.provider: "cloudProviderValue" } @@ -60,3 +60,103 @@ teardown: - match: { _source.host: null } - match: { _source.service.name: null } - match: { _source.service: null } + +--- +"Test rename special keys": + - do: + index: + index: ecs_namespacing_test + id: "rename_special_keys" + pipeline: "ecs_namespacing_pipeline" + body: { + "span": { + "id": "nestedSpanIdValue" + }, + "span.id": "topLevelSpanIdValue", + "log.level": "topLevelLogLevelValue", + "trace": { + "id": "traceIdValue" + }, + "trace.id": "topLevelTraceIdValue", + "message": "this is a message" + } + + - do: + get: + index: ecs_namespacing_test + id: "rename_special_keys" + - match: { _source.span_id: "nestedSpanIdValue" } + - match: { _source.severity_text: "topLevelLogLevelValue" } + - match: { _source.trace_id: "traceIdValue" } + - match: { _source.body.text: "this is a message" } + - match: { _source.span: null } + - match: { _source.span\.id: null } + - match: { _source.log\.level: null } + - match: { _source.trace: null } + - match: { _source.trace\.id: null } + - match: { _source.message: null } + +--- +"Test valid OTel document": + - do: + index: + index: ecs_namespacing_test + id: "valid_otel_document" + pipeline: "ecs_namespacing_pipeline" + body: { + "resource": { + "attributes": { + "foo": "bar" + } + }, + "scope": {}, + "attributes": { + "foo": "bar" + }, + "body": { + "text": "a string", + "structured": {} + }, + "foo": "bar" + } + + - do: + get: + index: ecs_namespacing_test + id: "valid_otel_document" + - match: { _source.resource.attributes.foo: "bar" } + - match: { _source.scope: {} } + - match: { _source.attributes.foo: "bar" } + - match: { _source.body.text: "a string" } + - match: { _source.body.structured: {} } + - match: { _source.foo: "bar" } + +--- +"Test invalid body field": + - do: + index: + index: ecs_namespacing_test + id: "invalid_scope_field" + pipeline: "ecs_namespacing_pipeline" + body: { + "resource": {}, + "scope": "invalid scope", + "body": { + "text": "test message", + "structured": { + "foo": "bar" + } + }, + "foo": "bar" + } + + - do: + get: + index: ecs_namespacing_test + id: "invalid_scope_field" + - match: { _source.attributes.scope: "invalid scope" } + - match: { _source.attributes.body\.text: "test message" } + - match: { _source.attributes.body\.structured\.foo: "bar" } + - match: { _source.attributes.foo: "bar" } + - match: { _source.body: null } + - match: { _source.scope: null } From 904c19c98b3c31bd0fbf2d407dd942668a368522 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 27 Mar 2025 14:12:20 +0200 Subject: [PATCH 05/45] Making scope non-mandatory --- .../ingest/ecs/EcsNamespacingProcessor.java | 2 +- .../ingest/ecs/EcsNamespacingProcessorTests.java | 11 +++-------- .../test/ingest-ecs/10_ecs_namespacing.yml | 14 ++++++++------ 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java index 86d745ecb21e2..8aa8df7e7a572 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java @@ -139,7 +139,7 @@ boolean isOTelDocument(Map source) { } Object scope = source.get(SCOPE_KEY); - if (scope instanceof Map == false) { + if (scope != null && scope instanceof Map == false) { return false; } diff --git a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java index eaf322b15ea67..b736236c78c53 100644 --- a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java +++ b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java @@ -24,11 +24,10 @@ public class EcsNamespacingProcessorTests extends ESTestCase { public void testIsOTelDocument_validMinimalOTelDocument() { Map document = new HashMap<>(); document.put("resource", new HashMap<>()); - document.put("scope", new HashMap<>()); assertTrue(processor.isOTelDocument(document)); } - public void testIsOTelDocument_validOTelDocumentWithAttributes() { + public void testIsOTelDocument_validOTelDocumentWithScopAndAttributes() { Map document = new HashMap<>(); document.put("attributes", new HashMap<>()); document.put("resource", new HashMap<>()); @@ -42,12 +41,6 @@ public void testIsOTelDocument_missingResource() { assertFalse(processor.isOTelDocument(document)); } - public void testIsOTelDocument_missingScope() { - Map document = new HashMap<>(); - document.put("resource", new HashMap<>()); - assertFalse(processor.isOTelDocument(document)); - } - public void testIsOTelDocument_resourceNotMap() { Map document = new HashMap<>(); document.put("resource", "not a map"); @@ -184,6 +177,7 @@ public void testExecute_nonOTelDocument_withExistingResource() { Map existingResource = new HashMap<>(); existingResource.put("existingKey", "existingValue"); document.put("resource", existingResource); + document.put("scope", "invalid scope"); document.put("key1", "value1"); IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); @@ -196,6 +190,7 @@ public void testExecute_nonOTelDocument_withExistingResource() { Map attributes = (Map) source.get("attributes"); assertEquals("value1", attributes.get("key1")); assertEquals("existingValue", attributes.get("resource.existingKey")); + assertEquals("invalid scope", attributes.get("scope")); Map resource = (Map) source.get("resource"); assertTrue(resource.containsKey("attributes")); diff --git a/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml b/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml index 6c09ba004d931..d0a96dc9d6221 100644 --- a/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml +++ b/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml @@ -136,13 +136,15 @@ teardown: - do: index: index: ecs_namespacing_test - id: "invalid_scope_field" + id: "invalid_body_field" pipeline: "ecs_namespacing_pipeline" body: { "resource": {}, - "scope": "invalid scope", + "scope": { + "foo": "bar" + }, "body": { - "text": "test message", + "text": 123, "structured": { "foo": "bar" } @@ -153,10 +155,10 @@ teardown: - do: get: index: ecs_namespacing_test - id: "invalid_scope_field" - - match: { _source.attributes.scope: "invalid scope" } - - match: { _source.attributes.body\.text: "test message" } + id: "invalid_body_field" + - match: { _source.attributes.body\.text: 123 } - match: { _source.attributes.body\.structured\.foo: "bar" } + - match: { _source.attributes.scope\.foo: "bar" } - match: { _source.attributes.foo: "bar" } - match: { _source.body: null } - match: { _source.scope: null } From bd75b063b9e69b92264dfe900c474e4b2cbc9329 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 27 Mar 2025 17:44:09 +0200 Subject: [PATCH 06/45] Minimize dependencies --- modules/ingest-ecs/src/main/java/module-info.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/ingest-ecs/src/main/java/module-info.java b/modules/ingest-ecs/src/main/java/module-info.java index 08129bbb0d3be..e2b3ec2861ff0 100644 --- a/modules/ingest-ecs/src/main/java/module-info.java +++ b/modules/ingest-ecs/src/main/java/module-info.java @@ -10,7 +10,4 @@ module org.elasticsearch.ingest.ecs { requires org.elasticsearch.base; requires org.elasticsearch.server; - requires org.elasticsearch.xcontent; - requires org.apache.logging.log4j; - requires org.elasticsearch.logging; } From 50f3c4d4f0e7637d208576e82f38164de4432ff0 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 27 Mar 2025 18:00:11 +0200 Subject: [PATCH 07/45] Extending REST tests --- .../test/ingest-ecs/10_ecs_namespacing.yml | 70 +++++++++++++++++-- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml b/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml index d0a96dc9d6221..8b8fcddb75228 100644 --- a/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml +++ b/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml @@ -24,19 +24,53 @@ teardown: body: { "agent.name": "agentNameValue", "agent": { - "type": "agentTypeValue" + "type": "agentTypeValue", + "deep": { + "nested": "nestedValue", + "scalar-array": [ + "arrayValue1", + "arrayValue2" + ], + "object-array": [ + { + "key1": "value1" + }, + { + "key2": "value2" + } + ] + }, + "scalar-array": [ + "arrayValue1", + "arrayValue2" + ] }, "cloud.provider": "cloudProviderValue", "cloud": { - "type": "cloudTypeValue" + "type": "cloudTypeValue" }, "host.name": "hostNameValue", "host": { - "type": "hostTypeValue" + "type": "hostTypeValue" }, "service.name": "serviceNameValue", "service": { - "type": "serviceTypeValue" + "type": "serviceTypeValue", + "deep": { + "nested": "nestedValue", + "scalar-array": [ + "arrayValue1", + "arrayValue2" + ], + "object-array": [ + { + "key1": "value1" + }, + { + "key2": "value2" + } + ] + }, } } @@ -46,12 +80,22 @@ teardown: id: "nested_and_flat_attributes" - match: { _source.resource.attributes.agent\.name: "agentNameValue" } - match: { _source.resource.attributes.agent\.type: "agentTypeValue" } + - match: { _source.resource.attributes.agent\.deep\.nested: "nestedValue" } + - match: { _source.resource.attributes.agent\.deep\.scalar-array.0: "arrayValue1" } + - match: { _source.resource.attributes.agent\.deep\.scalar-array.1: "arrayValue2" } + - match: { _source.resource.attributes.agent\.deep\.object-array.0.key1: "value1" } + - match: { _source.resource.attributes.agent\.deep\.object-array.1.key2: "value2" } - match: { _source.resource.attributes.cloud\.provider: "cloudProviderValue" } - match: { _source.resource.attributes.cloud\.type: "cloudTypeValue" } - match: { _source.resource.attributes.host\.name: "hostNameValue" } - match: { _source.resource.attributes.host\.type: "hostTypeValue" } - match: { _source.attributes.service\.name: "serviceNameValue" } - match: { _source.attributes.service\.type: "serviceTypeValue" } + - match: { _source.attributes.service\.deep\.nested: "nestedValue" } + - match: { _source.attributes.service\.deep\.scalar-array.0: "arrayValue1" } + - match: { _source.attributes.service\.deep\.scalar-array.1: "arrayValue2" } + - match: { _source.attributes.service\.deep\.object-array.0.key1: "value1" } + - match: { _source.attributes.service\.deep\.object-array.1.key2: "value2" } - match: { _source.agent.name: null } - match: { _source.agent: null } - match: { _source.cloud.provider: null } @@ -109,7 +153,9 @@ teardown: "foo": "bar" } }, - "scope": {}, + "scope": { + "foo": "bar" + }, "attributes": { "foo": "bar" }, @@ -117,6 +163,9 @@ teardown: "text": "a string", "structured": {} }, + "span_id": "spanIdValue", + "trace_id": "traceIdValue", + "severity_text": "severityTextValue", "foo": "bar" } @@ -125,10 +174,13 @@ teardown: index: ecs_namespacing_test id: "valid_otel_document" - match: { _source.resource.attributes.foo: "bar" } - - match: { _source.scope: {} } + - match: { _source.scope.foo: "bar" } - match: { _source.attributes.foo: "bar" } - match: { _source.body.text: "a string" } - match: { _source.body.structured: {} } + - match: { _source.span_id: "spanIdValue" } + - match: { _source.trace_id: "traceIdValue" } + - match: { _source.severity_text: "severityTextValue" } - match: { _source.foo: "bar" } --- @@ -149,6 +201,9 @@ teardown: "foo": "bar" } }, + "span_id": "spanIdValue", + "trace_id": "traceIdValue", + "severity_text": "severityTextValue", "foo": "bar" } @@ -159,6 +214,9 @@ teardown: - match: { _source.attributes.body\.text: 123 } - match: { _source.attributes.body\.structured\.foo: "bar" } - match: { _source.attributes.scope\.foo: "bar" } + - match: { _source.attributes.span_id: "spanIdValue" } + - match: { _source.attributes.trace_id: "traceIdValue" } + - match: { _source.attributes.severity_text: "severityTextValue" } - match: { _source.attributes.foo: "bar" } - match: { _source.body: null } - match: { _source.scope: null } From f68cf93d9b34643adbe26256eeb9a66a7205d077 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 27 Mar 2025 18:07:02 +0200 Subject: [PATCH 08/45] Update docs/changelog/125699.yaml --- docs/changelog/125699.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/125699.yaml diff --git a/docs/changelog/125699.yaml b/docs/changelog/125699.yaml new file mode 100644 index 0000000000000..7ba8e4bd2df82 --- /dev/null +++ b/docs/changelog/125699.yaml @@ -0,0 +1,5 @@ +pr: 125699 +summary: Adding `EcsNamespacingProcessor` +area: Ingest Node +type: feature +issues: [] From 79bf68338e30849f7b1d961b49db4d7679f0a18b Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Sun, 30 Mar 2025 10:05:24 +0300 Subject: [PATCH 09/45] instanceOf with pattern matching Co-authored-by: Lee Hinman --- .../ingest/ecs/EcsNamespacingProcessor.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java index 8aa8df7e7a572..8dffa917dc72b 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java @@ -150,15 +150,16 @@ boolean isOTelDocument(Map source) { Object body = source.get(BODY_KEY); if (body != null) { - if (body instanceof Map == false) { - return false; - } - Object bodyText = ((Map) body).get(TEXT_KEY); - if (bodyText != null && (bodyText instanceof String) == false) { + if (body instanceof Map bodyMap) { + Object bodyText = bodyMap.get(TEXT_KEY); + if (bodyText != null && (bodyText instanceof String) == false) { + return false; + } + Object bodyStructured = bodyMap.get(STRUCTURED_KEY); + return (bodyStructured instanceof String) == false; + } else { return false; } - Object bodyStructured = ((Map) body).get(STRUCTURED_KEY); - return (bodyStructured instanceof String) == false; } return true; } From dbc4d4ab05fbfe9ab1ffa6b22ad2c07473cf7d92 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Sun, 30 Mar 2025 10:06:12 +0300 Subject: [PATCH 10/45] instanceOf with pattern matching Co-authored-by: Lee Hinman --- .../elasticsearch/ingest/ecs/EcsNamespacingProcessor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java index 8dffa917dc72b..c6102dde01c9e 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java @@ -129,13 +129,13 @@ public IngestDocument execute(IngestDocument document) { @SuppressWarnings("unchecked") boolean isOTelDocument(Map source) { Object resource = source.get(RESOURCE_KEY); - if (resource instanceof Map == false) { - return false; - } else { - Object resourceAttributes = ((Map) resource).get(ATTRIBUTES_KEY); + if (resource instanceof Map resourceMap) { + Object resourceAttributes = resourceMap.get(ATTRIBUTES_KEY); if (resourceAttributes != null && (resourceAttributes instanceof Map) == false) { return false; } + } else { + return false; } Object scope = source.get(SCOPE_KEY); From d160fe641882db716e7836b641d7c9b8fb390306 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Sun, 30 Mar 2025 10:06:55 +0300 Subject: [PATCH 11/45] revert constants usage Co-authored-by: Lee Hinman --- .../ingest/ecs/EcsNamespacingProcessor.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java index c6102dde01c9e..23f38dcaefc59 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java @@ -195,12 +195,12 @@ private void renameSpecialKeys(IngestDocument document) { } private boolean shouldMoveToResourceAttributes(String key) { - return key.startsWith(AGENT_PREFIX) - || key.equals("agent") - || key.startsWith(CLOUD_PREFIX) - || key.equals("cloud") - || key.startsWith(HOST_PREFIX) - || key.equals("host"); + return key.startsWith(AGENT_PREFIX +".") + || key.equals(AGENT_PREFIX) + || key.startsWith(CLOUD_PREFIX + ".") + || key.equals(CLOUD_PREFIX) + || key.startsWith(HOST_PREFIX +".") + || key.equals(HOST_PREFIX); } public static final class Factory implements Processor.Factory { From 2b77e3c8f1d2695713d51eb218fa3a092f6c4f2c Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Sun, 30 Mar 2025 10:10:27 +0300 Subject: [PATCH 12/45] Complete review change proposals --- .../elasticsearch/ingest/ecs/EcsNamespacingProcessor.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java index 23f38dcaefc59..475554f308d8a 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java @@ -52,9 +52,9 @@ public class EcsNamespacingProcessor extends AbstractProcessor { KEEP_KEYS = Set.copyOf(keepKeys); } - private static final String AGENT_PREFIX = "agent."; - private static final String CLOUD_PREFIX = "cloud."; - private static final String HOST_PREFIX = "host."; + private static final String AGENT_PREFIX = "agent"; + private static final String CLOUD_PREFIX = "cloud"; + private static final String HOST_PREFIX = "host"; private static final String ATTRIBUTES_KEY = "attributes"; private static final String RESOURCE_KEY = "resource"; @@ -126,7 +126,6 @@ public IngestDocument execute(IngestDocument document) { return document; } - @SuppressWarnings("unchecked") boolean isOTelDocument(Map source) { Object resource = source.get(RESOURCE_KEY); if (resource instanceof Map resourceMap) { From 4773cf6941bac002ab982764a9481d7a6359719b Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Sun, 30 Mar 2025 10:14:33 +0300 Subject: [PATCH 13/45] fix typo in test name Co-authored-by: Lee Hinman --- .../elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java index b736236c78c53..d1a69467594e3 100644 --- a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java +++ b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java @@ -27,7 +27,7 @@ public void testIsOTelDocument_validMinimalOTelDocument() { assertTrue(processor.isOTelDocument(document)); } - public void testIsOTelDocument_validOTelDocumentWithScopAndAttributes() { + public void testIsOTelDocument_validOTelDocumentWithScopeAndAttributes() { Map document = new HashMap<>(); document.put("attributes", new HashMap<>()); document.put("resource", new HashMap<>()); From 054a76f39038fe87e3f3c62fb1e35c3cd35ef8c1 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sun, 30 Mar 2025 07:21:04 +0000 Subject: [PATCH 14/45] [CI] Auto commit changes from spotless --- .../org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java index 475554f308d8a..00a793ba060d6 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java @@ -194,11 +194,11 @@ private void renameSpecialKeys(IngestDocument document) { } private boolean shouldMoveToResourceAttributes(String key) { - return key.startsWith(AGENT_PREFIX +".") + return key.startsWith(AGENT_PREFIX + ".") || key.equals(AGENT_PREFIX) || key.startsWith(CLOUD_PREFIX + ".") || key.equals(CLOUD_PREFIX) - || key.startsWith(HOST_PREFIX +".") + || key.startsWith(HOST_PREFIX + ".") || key.equals(HOST_PREFIX); } From 4afacd0f05077900c75278e35c8e7b71fcfc9e38 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Sun, 30 Mar 2025 16:33:52 +0300 Subject: [PATCH 15/45] Applying review suggestions --- docs/changelog/125699.yaml | 2 +- modules/ingest-ecs/build.gradle | 2 +- ...ingPlugin.java => EcsNamespacePlugin.java} | 4 +- ...cessor.java => EcsNamespaceProcessor.java} | 72 +++++++++++++++++-- ...s.java => EcsNamespaceProcessorTests.java} | 36 +++++----- ...s_namespacing.yml => 10_ecs_namespace.yml} | 14 ++-- 6 files changed, 92 insertions(+), 38 deletions(-) rename modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/{EcsNamespacingPlugin.java => EcsNamespacePlugin.java} (82%) rename modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/{EcsNamespacingProcessor.java => EcsNamespaceProcessor.java} (65%) rename modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/{EcsNamespacingProcessorTests.java => EcsNamespaceProcessorTests.java} (93%) rename modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/{10_ecs_namespacing.yml => 10_ecs_namespace.yml} (95%) diff --git a/docs/changelog/125699.yaml b/docs/changelog/125699.yaml index 7ba8e4bd2df82..a5ea40340a14c 100644 --- a/docs/changelog/125699.yaml +++ b/docs/changelog/125699.yaml @@ -1,5 +1,5 @@ pr: 125699 -summary: Adding `EcsNamespacingProcessor` +summary: Adding `EcsNamespaceProcessor` area: Ingest Node type: feature issues: [] diff --git a/modules/ingest-ecs/build.gradle b/modules/ingest-ecs/build.gradle index c35c91b1d6a3d..3f118c84e4c0a 100644 --- a/modules/ingest-ecs/build.gradle +++ b/modules/ingest-ecs/build.gradle @@ -12,7 +12,7 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description = 'Ingest processor that applies ECS namespacing' - classname ='org.elasticsearch.ingest.ecs.EcsNamespacingPlugin' + classname ='org.elasticsearch.ingest.ecs.EcsNamespacePlugin' } restResources { diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingPlugin.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacePlugin.java similarity index 82% rename from modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingPlugin.java rename to modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacePlugin.java index 683fd9bcb41c9..6341d2830e93b 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingPlugin.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacePlugin.java @@ -15,10 +15,10 @@ import java.util.Map; -public class EcsNamespacingPlugin extends Plugin implements IngestPlugin { +public class EcsNamespacePlugin extends Plugin implements IngestPlugin { @Override public Map getProcessors(Processor.Parameters parameters) { - return Map.of(EcsNamespacingProcessor.TYPE, new EcsNamespacingProcessor.Factory()); + return Map.of(EcsNamespaceProcessor.TYPE, new EcsNamespaceProcessor.Factory()); } } diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java similarity index 65% rename from modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java rename to modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java index 475554f308d8a..1006037b79402 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java @@ -21,10 +21,28 @@ import java.util.Map; import java.util.Set; -public class EcsNamespacingProcessor extends AbstractProcessor { +/** + * This processor is responsible for transforming non-OpenTelemetry-compliant documents into a namespaced flavor of ECS + * that makes them compatible with OpenTelemetry. + * It DOES NOT translate the entire ECS schema into OpenTelemetry semantic conventions. + * + *

More specifically, this processor performs the following operations: + *

    + *
  • Renames specific ECS fields to their corresponding OpenTelemetry-compatible counterparts.
    • + *
  • Moves fields to the "attributes" and "resource.attributes" namespaces.
    • + *
  • Flattens the "attributes" and "resource.attributes" maps.
    • + *
+ * + *

If a document is identified as OpenTelemetry-compatible, no transformation is performed. + * @see org.elasticsearch.ingest.AbstractProcessor + */ +public class EcsNamespaceProcessor extends AbstractProcessor { - public static final String TYPE = "ecs_namespacing"; + public static final String TYPE = "ecs_namespace"; + /** + * Mapping of ECS field names to their corresponding OpenTelemetry-compatible counterparts. + */ private static final Map RENAME_KEYS = Map.of( "span.id", "span_id", @@ -36,11 +54,18 @@ public class EcsNamespacingProcessor extends AbstractProcessor { "trace_id" ); + /** + * A close-set of keys that should be kept at the top level of the processed document after applying the namespacing. + * In essence, these are the fields that should not be moved to the "attributes" or "resource.attributes" namespaces. + * Besides the @timestamp field, this set obviously contains the attributes and the resource fields, as well as the + * OpenTelemetry-compatible fields that are renamed by the processor. + */ private static final Set KEEP_KEYS; static { Set keepKeys = new HashSet<>(Set.of("@timestamp", "attributes", "resource")); Set renamedTopLevelFields = new HashSet<>(); for (String value : RENAME_KEYS.values()) { + // if the renamed field is nested, we only need to know the top level field int dotIndex = value.indexOf('.'); if (dotIndex != -1) { renamedTopLevelFields.add(value.substring(0, dotIndex)); @@ -63,7 +88,7 @@ public class EcsNamespacingProcessor extends AbstractProcessor { private static final String TEXT_KEY = "text"; private static final String STRUCTURED_KEY = "structured"; - EcsNamespacingProcessor(String tag, String description) { + EcsNamespaceProcessor(String tag, String description) { super(tag, description); } @@ -85,7 +110,9 @@ public IngestDocument execute(IngestDocument document) { Map newAttributes = new HashMap<>(); // The keep keys indicate the fields that should be kept at the top level later on when applying the namespacing. - // However, at this point we need to move their original values to the new attributes namespace, except for the @timestamp field. + // However, at this point we need to move their original values (if they exist) to the one of the new attributes namespaces, except + // for the @timestamp field. The assumption is that at this point the document is not OTel compliant, so even if a valid top + // level field is found, we assume that it does not bear the OTel semantics. for (String keepKey : KEEP_KEYS) { if (keepKey.equals("@timestamp")) { continue; @@ -126,7 +153,24 @@ public IngestDocument execute(IngestDocument document) { return document; } - boolean isOTelDocument(Map source) { + /** + * Checks if the given document is OpenTelemetry-compliant. + * + *

A document is considered OpenTelemetry-compliant if it meets the following criteria: + *

    + *
  • The "resource" field is present and is a map + *
  • The resource field either doesn't contain an "attributes" field, or the "attributes" field is a map.
    • + *
  • The "scope" field is either absent or a map.
    • + *
  • The "attributes" field is either absent or a map.
    • + *
  • The "body" field is either absent or a map.
    • + *
  • If exists, the "body" either doesn't contain a "text" field, or the "text" field is a string.
    • + *
  • If exists, the "body" either doesn't contain a "structured" field, or the "structured" field is not a string.
    • + *
+ * + * @param source the document to check + * @return {@code true} if the document is OpenTelemetry-compliant, {@code false} otherwise + */ + static boolean isOTelDocument(Map source) { Object resource = source.get(RESOURCE_KEY); if (resource instanceof Map resourceMap) { Object resourceAttributes = resourceMap.get(ATTRIBUTES_KEY); @@ -163,7 +207,21 @@ boolean isOTelDocument(Map source) { return true; } - private void renameSpecialKeys(IngestDocument document) { + /** + * Renames specific ECS keys in the given document to their OpenTelemetry-compatible counterparts, based on the {@code RENAME_KEYS} map. + * + *

This method performs the following operations: + *

    + *
  • For each key in the {@code RENAME_KEYS} map, it checks if a corresponding field exists in the document. If first looks for the + * field assuming dot notation for nested fields. If the field is not found, it looks for a top level field with a dotted name.
    • + *
  • If the field exists, it removes if from the document and adds a new field with the corresponding name from the {@code + * RENAME_KEYS} map and the same value.
    • + *
  • If the key is nested (contains dots), it recursively removes empty parent fields after renaming.
    • + *
+ * + * @param document the document to process + */ + static void renameSpecialKeys(IngestDocument document) { RENAME_KEYS.forEach((nonOtelName, otelName) -> { // first look assuming dot notation for nested fields Object value = document.getFieldValue(nonOtelName, Object.class, true); @@ -211,7 +269,7 @@ public Processor create( Map config, ProjectId projectId ) throws Exception { - return new EcsNamespacingProcessor(tag, description); + return new EcsNamespaceProcessor(tag, description); } } } diff --git a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java similarity index 93% rename from modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java rename to modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java index d1a69467594e3..288acd6d62f70 100644 --- a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespacingProcessorTests.java +++ b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java @@ -17,14 +17,14 @@ import java.util.Map; @SuppressWarnings("unchecked") -public class EcsNamespacingProcessorTests extends ESTestCase { +public class EcsNamespaceProcessorTests extends ESTestCase { - private final EcsNamespacingProcessor processor = new EcsNamespacingProcessor("test", "test processor"); + private final EcsNamespaceProcessor processor = new EcsNamespaceProcessor("test", "test processor"); public void testIsOTelDocument_validMinimalOTelDocument() { Map document = new HashMap<>(); document.put("resource", new HashMap<>()); - assertTrue(processor.isOTelDocument(document)); + assertTrue(EcsNamespaceProcessor.isOTelDocument(document)); } public void testIsOTelDocument_validOTelDocumentWithScopeAndAttributes() { @@ -32,20 +32,20 @@ public void testIsOTelDocument_validOTelDocumentWithScopeAndAttributes() { document.put("attributes", new HashMap<>()); document.put("resource", new HashMap<>()); document.put("scope", new HashMap<>()); - assertTrue(processor.isOTelDocument(document)); + assertTrue(EcsNamespaceProcessor.isOTelDocument(document)); } public void testIsOTelDocument_missingResource() { Map document = new HashMap<>(); document.put("scope", new HashMap<>()); - assertFalse(processor.isOTelDocument(document)); + assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); } public void testIsOTelDocument_resourceNotMap() { Map document = new HashMap<>(); document.put("resource", "not a map"); document.put("scope", new HashMap<>()); - assertFalse(processor.isOTelDocument(document)); + assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); } public void testIsOTelDocument_invalidResourceAttributes() { @@ -54,14 +54,14 @@ public void testIsOTelDocument_invalidResourceAttributes() { Map document = new HashMap<>(); document.put("resource", resource); document.put("scope", new HashMap<>()); - assertFalse(processor.isOTelDocument(document)); + assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); } public void testIsOTelDocument_scopeNotMap() { Map document = new HashMap<>(); document.put("resource", new HashMap<>()); document.put("scope", "not a map"); - assertFalse(processor.isOTelDocument(document)); + assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); } public void testIsOTelDocument_invalidAttributes() { @@ -69,7 +69,7 @@ public void testIsOTelDocument_invalidAttributes() { document.put("resource", new HashMap<>()); document.put("scope", new HashMap<>()); document.put("attributes", "not a map"); - assertFalse(processor.isOTelDocument(document)); + assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); } public void testIsOTelDocument_invalidBody() { @@ -77,7 +77,7 @@ public void testIsOTelDocument_invalidBody() { document.put("resource", new HashMap<>()); document.put("scope", new HashMap<>()); document.put("body", "not a map"); - assertFalse(processor.isOTelDocument(document)); + assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); } public void testIsOTelDocument_invalidBodyText() { @@ -87,7 +87,7 @@ public void testIsOTelDocument_invalidBodyText() { document.put("resource", new HashMap<>()); document.put("scope", new HashMap<>()); document.put("body", body); - assertFalse(processor.isOTelDocument(document)); + assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); } public void testIsOTelDocument_invalidBodyStructured() { @@ -97,7 +97,7 @@ public void testIsOTelDocument_invalidBodyStructured() { document.put("resource", new HashMap<>()); document.put("scope", new HashMap<>()); document.put("body", body); - assertFalse(processor.isOTelDocument(document)); + assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); } public void testIsOTelDocument_validBody() { @@ -108,7 +108,7 @@ public void testIsOTelDocument_validBody() { document.put("resource", new HashMap<>()); document.put("scope", new HashMap<>()); document.put("body", body); - assertTrue(processor.isOTelDocument(document)); + assertTrue(EcsNamespaceProcessor.isOTelDocument(document)); } public void testExecute_validOTelDocument() { @@ -210,7 +210,7 @@ public void testRenameSpecialKeys_nestedForm() { document.put("trace", trace); IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - processor.execute(ingestDocument); + EcsNamespaceProcessor.renameSpecialKeys(ingestDocument); Map source = ingestDocument.getSource(); assertEquals("spanIdValue", source.get("span_id")); @@ -219,8 +219,6 @@ public void testRenameSpecialKeys_nestedForm() { assertFalse(source.containsKey("log")); assertEquals("traceIdValue", source.get("trace_id")); assertFalse(source.containsKey("trace")); - assertTrue(source.containsKey("attributes")); - assertTrue(((Map) source.get("attributes")).isEmpty()); } public void testRenameSpecialKeys_topLevelDottedField() { @@ -231,7 +229,7 @@ public void testRenameSpecialKeys_topLevelDottedField() { document.put("message", "this is a message"); IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - processor.execute(ingestDocument); + EcsNamespaceProcessor.renameSpecialKeys(ingestDocument); Map source = ingestDocument.getSource(); assertEquals("spanIdValue", source.get("span_id")); @@ -240,8 +238,6 @@ public void testRenameSpecialKeys_topLevelDottedField() { Object body = source.get("body"); assertTrue(body instanceof Map); assertEquals("this is a message", ((Map) body).get("text")); - assertTrue(source.containsKey("attributes")); - assertTrue(((Map) source.get("attributes")).isEmpty()); assertFalse(document.containsKey("span.id")); assertFalse(document.containsKey("log.level")); assertFalse(document.containsKey("trace.id")); @@ -256,7 +252,7 @@ public void testRenameSpecialKeys_mixedForm() { document.put("span.id", "topLevelSpanIdValue"); IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - processor.execute(ingestDocument); + EcsNamespaceProcessor.renameSpecialKeys(ingestDocument); Map source = ingestDocument.getSource(); // nested form should take precedence diff --git a/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml b/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespace.yml similarity index 95% rename from modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml rename to modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespace.yml index 8b8fcddb75228..beacc600665bc 100644 --- a/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespacing.yml +++ b/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespace.yml @@ -2,16 +2,16 @@ setup: - do: ingest.put_pipeline: - id: "ecs_namespacing_pipeline" + id: "ecs_namespace_pipeline" body: processors: - - ecs_namespacing: {} + - ecs_namespace: {} --- teardown: - do: ingest.delete_pipeline: - id: "ecs_namespacing_pipeline" + id: "ecs_namespace_pipeline" ignore: 404 --- @@ -20,7 +20,7 @@ teardown: index: index: ecs_namespacing_test id: "nested_and_flat_attributes" - pipeline: "ecs_namespacing_pipeline" + pipeline: "ecs_namespace_pipeline" body: { "agent.name": "agentNameValue", "agent": { @@ -111,7 +111,7 @@ teardown: index: index: ecs_namespacing_test id: "rename_special_keys" - pipeline: "ecs_namespacing_pipeline" + pipeline: "ecs_namespace_pipeline" body: { "span": { "id": "nestedSpanIdValue" @@ -146,7 +146,7 @@ teardown: index: index: ecs_namespacing_test id: "valid_otel_document" - pipeline: "ecs_namespacing_pipeline" + pipeline: "ecs_namespace_pipeline" body: { "resource": { "attributes": { @@ -189,7 +189,7 @@ teardown: index: index: ecs_namespacing_test id: "invalid_body_field" - pipeline: "ecs_namespacing_pipeline" + pipeline: "ecs_namespace_pipeline" body: { "resource": {}, "scope": { From bef3643e77291044105a097b85a49bce261aab5a Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Thu, 3 Apr 2025 16:57:49 -0400 Subject: [PATCH 16/45] Silence a warning from Intellij --- .../org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java index 0f7d01f0ed916..8f9b911804298 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java @@ -268,7 +268,7 @@ public Processor create( String description, Map config, ProjectId projectId - ) throws Exception { + ) { return new EcsNamespaceProcessor(tag, description); } } From 49d439fb3e6cd6bb02c0b6fe4b22068d5ea296bb Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Thu, 3 Apr 2025 16:46:36 -0400 Subject: [PATCH 17/45] Rename this variable --- .../org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java index 8f9b911804298..5025233d4e021 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java @@ -263,7 +263,7 @@ private boolean shouldMoveToResourceAttributes(String key) { public static final class Factory implements Processor.Factory { @Override public Processor create( - Map processorFactories, + Map registry, String tag, String description, Map config, From e15984b89e2fc4be39de1d86d4de98bc3fd648e2 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Thu, 3 Apr 2025 16:58:09 -0400 Subject: [PATCH 18/45] Fix some typos and reflow a comment --- .../elasticsearch/ingest/ecs/EcsNamespaceProcessor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java index 5025233d4e021..cb6753392bd2a 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java @@ -55,7 +55,7 @@ public class EcsNamespaceProcessor extends AbstractProcessor { ); /** - * A close-set of keys that should be kept at the top level of the processed document after applying the namespacing. + * A closed-set of keys that should be kept at the top level of the processed document after applying the namespacing. * In essence, these are the fields that should not be moved to the "attributes" or "resource.attributes" namespaces. * Besides the @timestamp field, this set obviously contains the attributes and the resource fields, as well as the * OpenTelemetry-compatible fields that are renamed by the processor. @@ -212,10 +212,10 @@ static boolean isOTelDocument(Map source) { * *

This method performs the following operations: *

    - *
  • For each key in the {@code RENAME_KEYS} map, it checks if a corresponding field exists in the document. If first looks for the + *
  • For each key in the {@code RENAME_KEYS} map, it checks if a corresponding field exists in the document. It first looks for the * field assuming dot notation for nested fields. If the field is not found, it looks for a top level field with a dotted name.
    • - *
  • If the field exists, it removes if from the document and adds a new field with the corresponding name from the {@code - * RENAME_KEYS} map and the same value.
    • + *
  • If the field exists, it removes it from the document and adds a new field with the corresponding name from the + * {@code RENAME_KEYS} map and the same value.
    • *
  • If the key is nested (contains dots), it recursively removes empty parent fields after renaming.
    • *
* From e09fa14a119ec2b4ea6844f8ac5da1e0e6eb7ae9 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Thu, 3 Apr 2025 16:21:12 -0400 Subject: [PATCH 19/45] Use ofEntries for increased clarity --- .../ingest/ecs/EcsNamespaceProcessor.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java index cb6753392bd2a..f15cf4fc3cc22 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java @@ -21,6 +21,8 @@ import java.util.Map; import java.util.Set; +import static java.util.Map.entry; + /** * This processor is responsible for transforming non-OpenTelemetry-compliant documents into a namespaced flavor of ECS * that makes them compatible with OpenTelemetry. @@ -43,15 +45,11 @@ public class EcsNamespaceProcessor extends AbstractProcessor { /** * Mapping of ECS field names to their corresponding OpenTelemetry-compatible counterparts. */ - private static final Map RENAME_KEYS = Map.of( - "span.id", - "span_id", - "message", - "body.text", - "log.level", - "severity_text", - "trace.id", - "trace_id" + private static final Map RENAME_KEYS = Map.ofEntries( + entry("span.id", "span_id"), + entry("message", "body.text"), + entry("log.level", "severity_text"), + entry("trace.id", "trace_id") ); /** From b71fa3f9d639c8969753054eea8d733f9b17100f Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Thu, 3 Apr 2025 16:39:23 -0400 Subject: [PATCH 20/45] Save a rehash and some traversals --- .../ingest/ecs/EcsNamespaceProcessor.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java index f15cf4fc3cc22..077a865fa6cc6 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java @@ -11,7 +11,6 @@ import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.util.Maps; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; @@ -131,17 +130,20 @@ public IngestDocument execute(IngestDocument document) { renameSpecialKeys(document); // Iterate through all top level keys and move them to the appropriate namespace - for (String key : Sets.newHashSet(source.keySet())) { + final var sourceItr = source.entrySet().iterator(); + while (sourceItr.hasNext()) { + final var entry = sourceItr.next(); + final var key = entry.getKey(); + final var value = entry.getValue(); if (KEEP_KEYS.contains(key)) { continue; } if (shouldMoveToResourceAttributes(key)) { - Object value = source.remove(key); newResourceAttributes.put(key, value); } else { - Object value = source.remove(key); newAttributes.put(key, value); } + sourceItr.remove(); } // Flatten attributes From 98880cab2ff27422baf1a97c3b73d2d69f12e620 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 9 Apr 2025 16:21:29 -0400 Subject: [PATCH 21/45] This can be static --- .../org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java index 077a865fa6cc6..d57cfc044d10a 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java +++ b/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java @@ -251,7 +251,7 @@ static void renameSpecialKeys(IngestDocument document) { }); } - private boolean shouldMoveToResourceAttributes(String key) { + private static boolean shouldMoveToResourceAttributes(String key) { return key.startsWith(AGENT_PREFIX + ".") || key.equals(AGENT_PREFIX) || key.startsWith(CLOUD_PREFIX + ".") From d37587d6c6f86d68796018e2544aaddbfaaa8c6e Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 9 Apr 2025 16:31:01 -0400 Subject: [PATCH 22/45] Rely on mutability for these tests --- .../ecs/EcsNamespaceProcessorTests.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java index 288acd6d62f70..f649a2bc3f1c1 100644 --- a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java +++ b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java @@ -122,8 +122,8 @@ public void testExecute_validOTelDocument() { document.put("key1", "value1"); Map before = new HashMap<>(document); IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - IngestDocument result = processor.execute(ingestDocument); - assertEquals(before, result.getSource()); + processor.execute(ingestDocument); + assertEquals(before, ingestDocument.getSource()); } public void testExecute_nonOTelDocument() { @@ -132,9 +132,9 @@ public void testExecute_nonOTelDocument() { document.put("key2", "value2"); IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - IngestDocument result = processor.execute(ingestDocument); + processor.execute(ingestDocument); - Map source = result.getSource(); + Map source = ingestDocument.getSource(); assertTrue(source.containsKey("attributes")); assertTrue(source.containsKey("resource")); @@ -157,9 +157,9 @@ public void testExecute_nonOTelDocument_withExistingAttributes() { document.put("key1", "value1"); IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - IngestDocument result = processor.execute(ingestDocument); + processor.execute(ingestDocument); - Map source = result.getSource(); + Map source = ingestDocument.getSource(); assertTrue(source.containsKey("attributes")); assertTrue(source.containsKey("resource")); @@ -181,9 +181,9 @@ public void testExecute_nonOTelDocument_withExistingResource() { document.put("key1", "value1"); IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - IngestDocument result = processor.execute(ingestDocument); + processor.execute(ingestDocument); - Map source = result.getSource(); + Map source = ingestDocument.getSource(); assertTrue(source.containsKey("attributes")); assertTrue(source.containsKey("resource")); @@ -279,9 +279,9 @@ public void testExecute_moveToAttributeMaps() { document.put("service", service); IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - IngestDocument result = processor.execute(ingestDocument); + processor.execute(ingestDocument); - Map source = result.getSource(); + Map source = ingestDocument.getSource(); // all attributes should be flattened Map expectedResourceAttributes = Map.of( @@ -341,9 +341,9 @@ public void testExecute_deepFlattening() { IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - IngestDocument result = processor.execute(ingestDocument); + processor.execute(ingestDocument); - Map source = result.getSource(); + Map source = ingestDocument.getSource(); Map expectedResourceAttributes = Map.of("agent.name", "agentNameValue", "agent.details.type", "agentTypeValue"); @@ -377,9 +377,9 @@ public void testExecute_arraysNotFlattened() { IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - IngestDocument result = processor.execute(ingestDocument); + processor.execute(ingestDocument); - Map source = result.getSource(); + Map source = ingestDocument.getSource(); Map expectedResourceAttributes = Map.of("agent.name", "agentNameValue", "agent.array", agentArray); From 21f2f9e342a15fff4d888a090cbebd0af20ca34c Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 9 Apr 2025 16:33:57 -0400 Subject: [PATCH 23/45] Rename some variables --- .../ecs/EcsNamespaceProcessorTests.java | 348 +++++++++--------- 1 file changed, 174 insertions(+), 174 deletions(-) diff --git a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java index f649a2bc3f1c1..da3ddfef4df23 100644 --- a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java +++ b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java @@ -22,266 +22,266 @@ public class EcsNamespaceProcessorTests extends ESTestCase { private final EcsNamespaceProcessor processor = new EcsNamespaceProcessor("test", "test processor"); public void testIsOTelDocument_validMinimalOTelDocument() { - Map document = new HashMap<>(); - document.put("resource", new HashMap<>()); - assertTrue(EcsNamespaceProcessor.isOTelDocument(document)); + Map source = new HashMap<>(); + source.put("resource", new HashMap<>()); + assertTrue(EcsNamespaceProcessor.isOTelDocument(source)); } public void testIsOTelDocument_validOTelDocumentWithScopeAndAttributes() { - Map document = new HashMap<>(); - document.put("attributes", new HashMap<>()); - document.put("resource", new HashMap<>()); - document.put("scope", new HashMap<>()); - assertTrue(EcsNamespaceProcessor.isOTelDocument(document)); + Map source = new HashMap<>(); + source.put("attributes", new HashMap<>()); + source.put("resource", new HashMap<>()); + source.put("scope", new HashMap<>()); + assertTrue(EcsNamespaceProcessor.isOTelDocument(source)); } public void testIsOTelDocument_missingResource() { - Map document = new HashMap<>(); - document.put("scope", new HashMap<>()); - assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); + Map source = new HashMap<>(); + source.put("scope", new HashMap<>()); + assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); } public void testIsOTelDocument_resourceNotMap() { - Map document = new HashMap<>(); - document.put("resource", "not a map"); - document.put("scope", new HashMap<>()); - assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); + Map source = new HashMap<>(); + source.put("resource", "not a map"); + source.put("scope", new HashMap<>()); + assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidResourceAttributes() { Map resource = new HashMap<>(); resource.put("attributes", "not a map"); - Map document = new HashMap<>(); - document.put("resource", resource); - document.put("scope", new HashMap<>()); - assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); + Map source = new HashMap<>(); + source.put("resource", resource); + source.put("scope", new HashMap<>()); + assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); } public void testIsOTelDocument_scopeNotMap() { - Map document = new HashMap<>(); - document.put("resource", new HashMap<>()); - document.put("scope", "not a map"); - assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); + Map source = new HashMap<>(); + source.put("resource", new HashMap<>()); + source.put("scope", "not a map"); + assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidAttributes() { - Map document = new HashMap<>(); - document.put("resource", new HashMap<>()); - document.put("scope", new HashMap<>()); - document.put("attributes", "not a map"); - assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); + Map source = new HashMap<>(); + source.put("resource", new HashMap<>()); + source.put("scope", new HashMap<>()); + source.put("attributes", "not a map"); + assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidBody() { - Map document = new HashMap<>(); - document.put("resource", new HashMap<>()); - document.put("scope", new HashMap<>()); - document.put("body", "not a map"); - assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); + Map source = new HashMap<>(); + source.put("resource", new HashMap<>()); + source.put("scope", new HashMap<>()); + source.put("body", "not a map"); + assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidBodyText() { Map body = new HashMap<>(); body.put("text", 123); - Map document = new HashMap<>(); - document.put("resource", new HashMap<>()); - document.put("scope", new HashMap<>()); - document.put("body", body); - assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); + Map source = new HashMap<>(); + source.put("resource", new HashMap<>()); + source.put("scope", new HashMap<>()); + source.put("body", body); + assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidBodyStructured() { Map body = new HashMap<>(); body.put("structured", "a string"); - Map document = new HashMap<>(); - document.put("resource", new HashMap<>()); - document.put("scope", new HashMap<>()); - document.put("body", body); - assertFalse(EcsNamespaceProcessor.isOTelDocument(document)); + Map source = new HashMap<>(); + source.put("resource", new HashMap<>()); + source.put("scope", new HashMap<>()); + source.put("body", body); + assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); } public void testIsOTelDocument_validBody() { Map body = new HashMap<>(); body.put("text", "a string"); body.put("structured", new HashMap<>()); - Map document = new HashMap<>(); - document.put("resource", new HashMap<>()); - document.put("scope", new HashMap<>()); - document.put("body", body); - assertTrue(EcsNamespaceProcessor.isOTelDocument(document)); + Map source = new HashMap<>(); + source.put("resource", new HashMap<>()); + source.put("scope", new HashMap<>()); + source.put("body", body); + assertTrue(EcsNamespaceProcessor.isOTelDocument(source)); } public void testExecute_validOTelDocument() { - Map document = new HashMap<>(); - document.put("resource", new HashMap<>()); - document.put("scope", new HashMap<>()); + Map source = new HashMap<>(); + source.put("resource", new HashMap<>()); + source.put("scope", new HashMap<>()); Map body = new HashMap<>(); body.put("text", "a string"); body.put("structured", new HashMap<>()); - document.put("body", body); - document.put("key1", "value1"); - Map before = new HashMap<>(document); - IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - processor.execute(ingestDocument); - assertEquals(before, ingestDocument.getSource()); + source.put("body", body); + source.put("key1", "value1"); + Map before = new HashMap<>(source); + IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); + processor.execute(document); + assertEquals(before, document.getSource()); } public void testExecute_nonOTelDocument() { - Map document = new HashMap<>(); - document.put("key1", "value1"); - document.put("key2", "value2"); - IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + Map source = new HashMap<>(); + source.put("key1", "value1"); + source.put("key2", "value2"); + IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - processor.execute(ingestDocument); + processor.execute(document); - Map source = ingestDocument.getSource(); - assertTrue(source.containsKey("attributes")); - assertTrue(source.containsKey("resource")); + Map result = document.getSource(); + assertTrue(result.containsKey("attributes")); + assertTrue(result.containsKey("resource")); - Map attributes = (Map) source.get("attributes"); + Map attributes = (Map) result.get("attributes"); assertEquals("value1", attributes.get("key1")); assertEquals("value2", attributes.get("key2")); - assertFalse(document.containsKey("key1")); - assertFalse(document.containsKey("key2")); + assertFalse(source.containsKey("key1")); + assertFalse(source.containsKey("key2")); - Map resource = (Map) source.get("resource"); + Map resource = (Map) result.get("resource"); assertTrue(resource.containsKey("attributes")); assertTrue(((Map) resource.get("attributes")).isEmpty()); } public void testExecute_nonOTelDocument_withExistingAttributes() { - Map document = new HashMap<>(); + Map source = new HashMap<>(); Map existingAttributes = new HashMap<>(); existingAttributes.put("existingKey", "existingValue"); - document.put("attributes", existingAttributes); - document.put("key1", "value1"); - IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + source.put("attributes", existingAttributes); + source.put("key1", "value1"); + IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - processor.execute(ingestDocument); + processor.execute(document); - Map source = ingestDocument.getSource(); - assertTrue(source.containsKey("attributes")); - assertTrue(source.containsKey("resource")); + Map result = document.getSource(); + assertTrue(result.containsKey("attributes")); + assertTrue(result.containsKey("resource")); - Map attributes = (Map) source.get("attributes"); + Map attributes = (Map) result.get("attributes"); assertEquals("existingValue", attributes.get("attributes.existingKey")); assertEquals("value1", attributes.get("key1")); - Map resource = (Map) source.get("resource"); + Map resource = (Map) result.get("resource"); assertTrue(resource.containsKey("attributes")); assertTrue(((Map) resource.get("attributes")).isEmpty()); } public void testExecute_nonOTelDocument_withExistingResource() { - Map document = new HashMap<>(); + Map source = new HashMap<>(); Map existingResource = new HashMap<>(); existingResource.put("existingKey", "existingValue"); - document.put("resource", existingResource); - document.put("scope", "invalid scope"); - document.put("key1", "value1"); - IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + source.put("resource", existingResource); + source.put("scope", "invalid scope"); + source.put("key1", "value1"); + IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - processor.execute(ingestDocument); + processor.execute(document); - Map source = ingestDocument.getSource(); - assertTrue(source.containsKey("attributes")); - assertTrue(source.containsKey("resource")); + Map result = document.getSource(); + assertTrue(result.containsKey("attributes")); + assertTrue(result.containsKey("resource")); - Map attributes = (Map) source.get("attributes"); + Map attributes = (Map) result.get("attributes"); assertEquals("value1", attributes.get("key1")); assertEquals("existingValue", attributes.get("resource.existingKey")); assertEquals("invalid scope", attributes.get("scope")); - Map resource = (Map) source.get("resource"); + Map resource = (Map) result.get("resource"); assertTrue(resource.containsKey("attributes")); assertTrue(((Map) resource.get("attributes")).isEmpty()); } public void testRenameSpecialKeys_nestedForm() { - Map document = new HashMap<>(); + Map source = new HashMap<>(); Map span = new HashMap<>(); span.put("id", "spanIdValue"); - document.put("span", span); + source.put("span", span); Map log = new HashMap<>(); log.put("level", "logLevelValue"); - document.put("log", log); + source.put("log", log); Map trace = new HashMap<>(); trace.put("id", "traceIdValue"); - document.put("trace", trace); - IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - - EcsNamespaceProcessor.renameSpecialKeys(ingestDocument); - - Map source = ingestDocument.getSource(); - assertEquals("spanIdValue", source.get("span_id")); - assertFalse(source.containsKey("span")); - assertEquals("logLevelValue", source.get("severity_text")); - assertFalse(source.containsKey("log")); - assertEquals("traceIdValue", source.get("trace_id")); - assertFalse(source.containsKey("trace")); + source.put("trace", trace); + IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); + + EcsNamespaceProcessor.renameSpecialKeys(document); + + Map result = document.getSource(); + assertEquals("spanIdValue", result.get("span_id")); + assertFalse(result.containsKey("span")); + assertEquals("logLevelValue", result.get("severity_text")); + assertFalse(result.containsKey("log")); + assertEquals("traceIdValue", result.get("trace_id")); + assertFalse(result.containsKey("trace")); } public void testRenameSpecialKeys_topLevelDottedField() { - Map document = new HashMap<>(); - document.put("span.id", "spanIdValue"); - document.put("log.level", "logLevelValue"); - document.put("trace.id", "traceIdValue"); - document.put("message", "this is a message"); - IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); - - EcsNamespaceProcessor.renameSpecialKeys(ingestDocument); - - Map source = ingestDocument.getSource(); - assertEquals("spanIdValue", source.get("span_id")); - assertEquals("logLevelValue", source.get("severity_text")); - assertEquals("traceIdValue", source.get("trace_id")); - Object body = source.get("body"); + Map source = new HashMap<>(); + source.put("span.id", "spanIdValue"); + source.put("log.level", "logLevelValue"); + source.put("trace.id", "traceIdValue"); + source.put("message", "this is a message"); + IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); + + EcsNamespaceProcessor.renameSpecialKeys(document); + + Map result = document.getSource(); + assertEquals("spanIdValue", result.get("span_id")); + assertEquals("logLevelValue", result.get("severity_text")); + assertEquals("traceIdValue", result.get("trace_id")); + Object body = result.get("body"); assertTrue(body instanceof Map); assertEquals("this is a message", ((Map) body).get("text")); - assertFalse(document.containsKey("span.id")); - assertFalse(document.containsKey("log.level")); - assertFalse(document.containsKey("trace.id")); - assertFalse(document.containsKey("message")); + assertFalse(source.containsKey("span.id")); + assertFalse(source.containsKey("log.level")); + assertFalse(source.containsKey("trace.id")); + assertFalse(source.containsKey("message")); } public void testRenameSpecialKeys_mixedForm() { - Map document = new HashMap<>(); + Map source = new HashMap<>(); Map span = new HashMap<>(); span.put("id", "nestedSpanIdValue"); - document.put("span", span); - document.put("span.id", "topLevelSpanIdValue"); - IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + source.put("span", span); + source.put("span.id", "topLevelSpanIdValue"); + IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - EcsNamespaceProcessor.renameSpecialKeys(ingestDocument); + EcsNamespaceProcessor.renameSpecialKeys(document); - Map source = ingestDocument.getSource(); + Map result = document.getSource(); // nested form should take precedence - assertEquals("nestedSpanIdValue", source.get("span_id")); + assertEquals("nestedSpanIdValue", result.get("span_id")); } public void testExecute_moveToAttributeMaps() { - Map document = new HashMap<>(); - document.put("agent.name", "agentNameValue"); + Map source = new HashMap<>(); + source.put("agent.name", "agentNameValue"); Map agent = new HashMap<>(); agent.put("type", "agentTypeValue"); - document.put("agent", agent); - document.put("cloud.provider", "cloudProviderValue"); + source.put("agent", agent); + source.put("cloud.provider", "cloudProviderValue"); Map cloud = new HashMap<>(); cloud.put("type", "cloudTypeValue"); - document.put("cloud", cloud); - document.put("host.name", "hostNameValue"); + source.put("cloud", cloud); + source.put("host.name", "hostNameValue"); Map host = new HashMap<>(); host.put("type", "hostTypeValue"); - document.put("host", host); - document.put("service.name", "serviceNameValue"); + source.put("host", host); + source.put("service.name", "serviceNameValue"); Map service = new HashMap<>(); service.put("type", "serviceTypeValue"); - document.put("service", service); - IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + source.put("service", service); + IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - processor.execute(ingestDocument); + processor.execute(document); - Map source = ingestDocument.getSource(); + Map result = document.getSource(); // all attributes should be flattened Map expectedResourceAttributes = Map.of( @@ -299,100 +299,100 @@ public void testExecute_moveToAttributeMaps() { "hostTypeValue" ); - assertTrue(source.containsKey("resource")); - Map resource = (Map) source.get("resource"); + assertTrue(result.containsKey("resource")); + Map resource = (Map) result.get("resource"); Map resourceAttributes = (Map) resource.get("attributes"); assertEquals(expectedResourceAttributes, resourceAttributes); assertNull(resourceAttributes.get("agent")); assertNull(resourceAttributes.get("cloud")); assertNull(resourceAttributes.get("host")); - assertFalse(document.containsKey("agent.name")); - assertFalse(document.containsKey("agent")); - assertFalse(document.containsKey("cloud.provider")); - assertFalse(document.containsKey("cloud")); - assertFalse(document.containsKey("host.name")); - assertFalse(document.containsKey("host")); + assertFalse(source.containsKey("agent.name")); + assertFalse(source.containsKey("agent")); + assertFalse(source.containsKey("cloud.provider")); + assertFalse(source.containsKey("cloud")); + assertFalse(source.containsKey("host.name")); + assertFalse(source.containsKey("host")); Map expectedAttributes = Map.of("service.name", "serviceNameValue", "service.type", "serviceTypeValue"); - assertTrue(source.containsKey("attributes")); - Map attributes = (Map) source.get("attributes"); + assertTrue(result.containsKey("attributes")); + Map attributes = (Map) result.get("attributes"); assertEquals(expectedAttributes, attributes); assertNull(attributes.get("service")); - assertFalse(document.containsKey("service.name")); - assertFalse(document.containsKey("service")); + assertFalse(source.containsKey("service.name")); + assertFalse(source.containsKey("service")); } public void testExecute_deepFlattening() { - Map document = new HashMap<>(); + Map source = new HashMap<>(); Map nestedAgent = new HashMap<>(); nestedAgent.put("name", "agentNameValue"); Map deeperAgent = new HashMap<>(); deeperAgent.put("type", "agentTypeValue"); nestedAgent.put("details", deeperAgent); - document.put("agent", nestedAgent); + source.put("agent", nestedAgent); Map nestedService = new HashMap<>(); nestedService.put("name", "serviceNameValue"); Map deeperService = new HashMap<>(); deeperService.put("type", "serviceTypeValue"); nestedService.put("details", deeperService); - document.put("service", nestedService); + source.put("service", nestedService); - IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - processor.execute(ingestDocument); + processor.execute(document); - Map source = ingestDocument.getSource(); + Map result = document.getSource(); Map expectedResourceAttributes = Map.of("agent.name", "agentNameValue", "agent.details.type", "agentTypeValue"); - assertTrue(source.containsKey("resource")); - Map resource = (Map) source.get("resource"); + assertTrue(result.containsKey("resource")); + Map resource = (Map) result.get("resource"); Map resourceAttributes = (Map) resource.get("attributes"); assertEquals(expectedResourceAttributes, resourceAttributes); assertNull(resource.get("agent")); Map expectedAttributes = Map.of("service.name", "serviceNameValue", "service.details.type", "serviceTypeValue"); - assertTrue(source.containsKey("attributes")); - Map attributes = (Map) source.get("attributes"); + assertTrue(result.containsKey("attributes")); + Map attributes = (Map) result.get("attributes"); assertEquals(expectedAttributes, attributes); assertNull(attributes.get("service")); } public void testExecute_arraysNotFlattened() { - Map document = new HashMap<>(); + Map source = new HashMap<>(); Map nestedAgent = new HashMap<>(); nestedAgent.put("name", "agentNameValue"); List agentArray = List.of("value1", "value2"); nestedAgent.put("array", agentArray); - document.put("agent", nestedAgent); + source.put("agent", nestedAgent); Map nestedService = new HashMap<>(); nestedService.put("name", "serviceNameValue"); List serviceArray = List.of("value1", "value2"); nestedService.put("array", serviceArray); - document.put("service", nestedService); + source.put("service", nestedService); - IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); + IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - processor.execute(ingestDocument); + processor.execute(document); - Map source = ingestDocument.getSource(); + Map result = document.getSource(); Map expectedResourceAttributes = Map.of("agent.name", "agentNameValue", "agent.array", agentArray); - assertTrue(source.containsKey("resource")); - Map resource = (Map) source.get("resource"); + assertTrue(result.containsKey("resource")); + Map resource = (Map) result.get("resource"); Map resourceAttributes = (Map) resource.get("attributes"); assertEquals(expectedResourceAttributes, resourceAttributes); assertNull(resource.get("agent")); Map expectedAttributes = Map.of("service.name", "serviceNameValue", "service.array", serviceArray); - assertTrue(source.containsKey("attributes")); - Map attributes = (Map) source.get("attributes"); + assertTrue(result.containsKey("attributes")); + Map attributes = (Map) result.get("attributes"); assertEquals(expectedAttributes, attributes); assertNull(attributes.get("service")); } From b0037c3efe331875cef43248442249287d0a4e25 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 9 Apr 2025 16:53:30 -0400 Subject: [PATCH 24/45] Drop the top-level warnings suppression --- .../ecs/EcsNamespaceProcessorTests.java | 54 +++++++++++-------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java index da3ddfef4df23..64fa22d8bb1c6 100644 --- a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java +++ b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java @@ -16,7 +16,6 @@ import java.util.List; import java.util.Map; -@SuppressWarnings("unchecked") public class EcsNamespaceProcessorTests extends ESTestCase { private final EcsNamespaceProcessor processor = new EcsNamespaceProcessor("test", "test processor"); @@ -138,15 +137,16 @@ public void testExecute_nonOTelDocument() { assertTrue(result.containsKey("attributes")); assertTrue(result.containsKey("resource")); - Map attributes = (Map) result.get("attributes"); + Map attributes = get(result, "attributes"); assertEquals("value1", attributes.get("key1")); assertEquals("value2", attributes.get("key2")); assertFalse(source.containsKey("key1")); assertFalse(source.containsKey("key2")); - Map resource = (Map) result.get("resource"); + Map resource = get(result, "resource"); assertTrue(resource.containsKey("attributes")); - assertTrue(((Map) resource.get("attributes")).isEmpty()); + Map resourceAttributes = get(resource, "attributes"); + assertTrue(resourceAttributes.isEmpty()); } public void testExecute_nonOTelDocument_withExistingAttributes() { @@ -163,13 +163,14 @@ public void testExecute_nonOTelDocument_withExistingAttributes() { assertTrue(result.containsKey("attributes")); assertTrue(result.containsKey("resource")); - Map attributes = (Map) result.get("attributes"); + Map attributes = get(result, "attributes"); assertEquals("existingValue", attributes.get("attributes.existingKey")); assertEquals("value1", attributes.get("key1")); - Map resource = (Map) result.get("resource"); + Map resource = get(result, "resource"); assertTrue(resource.containsKey("attributes")); - assertTrue(((Map) resource.get("attributes")).isEmpty()); + Map resourceAttributes = get(resource, "attributes"); + assertTrue(resourceAttributes.isEmpty()); } public void testExecute_nonOTelDocument_withExistingResource() { @@ -187,14 +188,15 @@ public void testExecute_nonOTelDocument_withExistingResource() { assertTrue(result.containsKey("attributes")); assertTrue(result.containsKey("resource")); - Map attributes = (Map) result.get("attributes"); + Map attributes = get(result, "attributes"); assertEquals("value1", attributes.get("key1")); assertEquals("existingValue", attributes.get("resource.existingKey")); assertEquals("invalid scope", attributes.get("scope")); - Map resource = (Map) result.get("resource"); + Map resource = get(result, "resource"); assertTrue(resource.containsKey("attributes")); - assertTrue(((Map) resource.get("attributes")).isEmpty()); + Map resourceAttributes = get(resource, "attributes"); + assertTrue(resourceAttributes.isEmpty()); } public void testRenameSpecialKeys_nestedForm() { @@ -235,9 +237,9 @@ public void testRenameSpecialKeys_topLevelDottedField() { assertEquals("spanIdValue", result.get("span_id")); assertEquals("logLevelValue", result.get("severity_text")); assertEquals("traceIdValue", result.get("trace_id")); - Object body = result.get("body"); - assertTrue(body instanceof Map); - assertEquals("this is a message", ((Map) body).get("text")); + Map body = get(result, "body"); + String text = get(body, "text"); + assertEquals("this is a message", text); assertFalse(source.containsKey("span.id")); assertFalse(source.containsKey("log.level")); assertFalse(source.containsKey("trace.id")); @@ -300,8 +302,8 @@ public void testExecute_moveToAttributeMaps() { ); assertTrue(result.containsKey("resource")); - Map resource = (Map) result.get("resource"); - Map resourceAttributes = (Map) resource.get("attributes"); + Map resource = get(result, "resource"); + Map resourceAttributes = get(resource, "attributes"); assertEquals(expectedResourceAttributes, resourceAttributes); assertNull(resourceAttributes.get("agent")); assertNull(resourceAttributes.get("cloud")); @@ -316,7 +318,7 @@ public void testExecute_moveToAttributeMaps() { Map expectedAttributes = Map.of("service.name", "serviceNameValue", "service.type", "serviceTypeValue"); assertTrue(result.containsKey("attributes")); - Map attributes = (Map) result.get("attributes"); + Map attributes = get(result, "attributes"); assertEquals(expectedAttributes, attributes); assertNull(attributes.get("service")); assertFalse(source.containsKey("service.name")); @@ -348,15 +350,15 @@ public void testExecute_deepFlattening() { Map expectedResourceAttributes = Map.of("agent.name", "agentNameValue", "agent.details.type", "agentTypeValue"); assertTrue(result.containsKey("resource")); - Map resource = (Map) result.get("resource"); - Map resourceAttributes = (Map) resource.get("attributes"); + Map resource = get(result, "resource"); + Map resourceAttributes = get(resource, "attributes"); assertEquals(expectedResourceAttributes, resourceAttributes); assertNull(resource.get("agent")); Map expectedAttributes = Map.of("service.name", "serviceNameValue", "service.details.type", "serviceTypeValue"); assertTrue(result.containsKey("attributes")); - Map attributes = (Map) result.get("attributes"); + Map attributes = get(result, "attributes"); assertEquals(expectedAttributes, attributes); assertNull(attributes.get("service")); } @@ -384,16 +386,24 @@ public void testExecute_arraysNotFlattened() { Map expectedResourceAttributes = Map.of("agent.name", "agentNameValue", "agent.array", agentArray); assertTrue(result.containsKey("resource")); - Map resource = (Map) result.get("resource"); - Map resourceAttributes = (Map) resource.get("attributes"); + Map resource = get(result, "resource"); + Map resourceAttributes = get(resource, "attributes"); assertEquals(expectedResourceAttributes, resourceAttributes); assertNull(resource.get("agent")); Map expectedAttributes = Map.of("service.name", "serviceNameValue", "service.array", serviceArray); assertTrue(result.containsKey("attributes")); - Map attributes = (Map) result.get("attributes"); + Map attributes = get(result, "attributes"); assertEquals(expectedAttributes, attributes); assertNull(attributes.get("service")); } + + /** + * A utility function for getting a key from a map and casting the result. + */ + @SuppressWarnings("unchecked") + private static T get(Map context, String key) { + return (T) context.get(key); + } } From 65f7ea2a2f613b2a5e3d62d69f9c12880e88cf77 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 9 Apr 2025 17:11:42 -0400 Subject: [PATCH 25/45] Rewrite this test Assert that it's the sameInstance, and use an immutable map of immutable maps in order to be sure that nothing changed. --- .../ecs/EcsNamespaceProcessorTests.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java index 64fa22d8bb1c6..3fcfc91de8316 100644 --- a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java +++ b/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java @@ -16,6 +16,9 @@ import java.util.List; import java.util.Map; +import static java.util.Map.entry; +import static org.hamcrest.Matchers.sameInstance; + public class EcsNamespaceProcessorTests extends ESTestCase { private final EcsNamespaceProcessor processor = new EcsNamespaceProcessor("test", "test processor"); @@ -111,18 +114,15 @@ public void testIsOTelDocument_validBody() { } public void testExecute_validOTelDocument() { - Map source = new HashMap<>(); - source.put("resource", new HashMap<>()); - source.put("scope", new HashMap<>()); - Map body = new HashMap<>(); - body.put("text", "a string"); - body.put("structured", new HashMap<>()); - source.put("body", body); - source.put("key1", "value1"); - Map before = new HashMap<>(source); + Map source = Map.ofEntries( + entry("resource", Map.of()), + entry("scope", Map.of()), + entry("body", Map.of("text", "a string", "structured", Map.of())), + entry("key1", "value1") + ); IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); processor.execute(document); - assertEquals(before, document.getSource()); + assertThat(source, sameInstance(document.getSource())); } public void testExecute_nonOTelDocument() { From f9421ac78e1d3ad660b5f16ef47f80850dfc0629 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 25 Apr 2025 16:26:27 -0400 Subject: [PATCH 26/45] Drop yaml-rest-compat-test There aren't any tests of this on 8.19, so there's nothing to test this against in a yaml-rest-compat-test way. --- modules/ingest-ecs/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/ingest-ecs/build.gradle b/modules/ingest-ecs/build.gradle index 3f118c84e4c0a..1beb683e2a3f4 100644 --- a/modules/ingest-ecs/build.gradle +++ b/modules/ingest-ecs/build.gradle @@ -8,7 +8,6 @@ */ apply plugin: 'elasticsearch.internal-yaml-rest-test' -apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description = 'Ingest processor that applies ECS namespacing' From b2dd61db15eb75bc41745ab00adcbbea95b0c66b Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Mon, 28 Apr 2025 11:55:02 +0300 Subject: [PATCH 27/45] Refactor ECS Namespacing to Normalize to OTel --- docs/changelog/125699.yaml | 2 +- .../{ingest-ecs => ingest-otel}/build.gradle | 4 +-- .../src/main/java/module-info.java | 2 +- .../ingest/otel/NormalizeToOTelPlugin.java} | 6 ++-- .../otel/NormalizeToOTelProcessor.java} | 10 +++--- .../otel/NormalizeToOTelProcessorTests.java} | 34 +++++++++---------- .../IngestOtelClientYamlTestSuiteIT.java} | 8 ++--- .../ingest-otel/10_normalize_to_otel.yml} | 30 ++++++++-------- 8 files changed, 48 insertions(+), 48 deletions(-) rename modules/{ingest-ecs => ingest-otel}/build.gradle (79%) rename modules/{ingest-ecs => ingest-otel}/src/main/java/module-info.java (93%) rename modules/{ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacePlugin.java => ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelPlugin.java} (77%) rename modules/{ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java => ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java} (97%) rename modules/{ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java => ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java} (93%) rename modules/{ingest-ecs/src/yamlRestTest/java/org/elasticsearch/ingest/ecs/IngestEcsClientYamlTestSuiteIT.java => ingest-otel/src/yamlRestTest/java/org/elasticsearch/ingest/otel/IngestOtelClientYamlTestSuiteIT.java} (82%) rename modules/{ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespace.yml => ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_to_otel.yml} (91%) diff --git a/docs/changelog/125699.yaml b/docs/changelog/125699.yaml index a5ea40340a14c..af3b6d5265b0d 100644 --- a/docs/changelog/125699.yaml +++ b/docs/changelog/125699.yaml @@ -1,5 +1,5 @@ pr: 125699 -summary: Adding `EcsNamespaceProcessor` +summary: Adding `NormalizeToOTelProcessor` area: Ingest Node type: feature issues: [] diff --git a/modules/ingest-ecs/build.gradle b/modules/ingest-otel/build.gradle similarity index 79% rename from modules/ingest-ecs/build.gradle rename to modules/ingest-otel/build.gradle index 1beb683e2a3f4..7818ae32e0eda 100644 --- a/modules/ingest-ecs/build.gradle +++ b/modules/ingest-otel/build.gradle @@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test' esplugin { - description = 'Ingest processor that applies ECS namespacing' - classname ='org.elasticsearch.ingest.ecs.EcsNamespacePlugin' + description = 'Ingest processor that normalizes ECS documents to OpenTelemetry-compatible namespaces' + classname ='org.elasticsearch.ingest.otel.NormalizeToOTelPlugin' } restResources { diff --git a/modules/ingest-ecs/src/main/java/module-info.java b/modules/ingest-otel/src/main/java/module-info.java similarity index 93% rename from modules/ingest-ecs/src/main/java/module-info.java rename to modules/ingest-otel/src/main/java/module-info.java index e2b3ec2861ff0..20b349d930c85 100644 --- a/modules/ingest-ecs/src/main/java/module-info.java +++ b/modules/ingest-otel/src/main/java/module-info.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -module org.elasticsearch.ingest.ecs { +module org.elasticsearch.ingest.otel { requires org.elasticsearch.base; requires org.elasticsearch.server; } diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacePlugin.java b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelPlugin.java similarity index 77% rename from modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacePlugin.java rename to modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelPlugin.java index 6341d2830e93b..205795de435e6 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespacePlugin.java +++ b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelPlugin.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.ingest.ecs; +package org.elasticsearch.ingest.otel; import org.elasticsearch.ingest.Processor; import org.elasticsearch.plugins.IngestPlugin; @@ -15,10 +15,10 @@ import java.util.Map; -public class EcsNamespacePlugin extends Plugin implements IngestPlugin { +public class NormalizeToOTelPlugin extends Plugin implements IngestPlugin { @Override public Map getProcessors(Processor.Parameters parameters) { - return Map.of(EcsNamespaceProcessor.TYPE, new EcsNamespaceProcessor.Factory()); + return Map.of(NormalizeToOTelProcessor.TYPE, new NormalizeToOTelProcessor.Factory()); } } diff --git a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java similarity index 97% rename from modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java rename to modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java index d57cfc044d10a..3ced62d19bbf6 100644 --- a/modules/ingest-ecs/src/main/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessor.java +++ b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.ingest.ecs; +package org.elasticsearch.ingest.otel; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.util.Maps; @@ -37,9 +37,9 @@ *

If a document is identified as OpenTelemetry-compatible, no transformation is performed. * @see org.elasticsearch.ingest.AbstractProcessor */ -public class EcsNamespaceProcessor extends AbstractProcessor { +public class NormalizeToOTelProcessor extends AbstractProcessor { - public static final String TYPE = "ecs_namespace"; + public static final String TYPE = "normalize_to_otel"; /** * Mapping of ECS field names to their corresponding OpenTelemetry-compatible counterparts. @@ -85,7 +85,7 @@ public class EcsNamespaceProcessor extends AbstractProcessor { private static final String TEXT_KEY = "text"; private static final String STRUCTURED_KEY = "structured"; - EcsNamespaceProcessor(String tag, String description) { + NormalizeToOTelProcessor(String tag, String description) { super(tag, description); } @@ -269,7 +269,7 @@ public Processor create( Map config, ProjectId projectId ) { - return new EcsNamespaceProcessor(tag, description); + return new NormalizeToOTelProcessor(tag, description); } } } diff --git a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java similarity index 93% rename from modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java rename to modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java index 3fcfc91de8316..19c2199799159 100644 --- a/modules/ingest-ecs/src/test/java/org/elasticsearch/ingest/ecs/EcsNamespaceProcessorTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.ingest.ecs; +package org.elasticsearch.ingest.otel; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.ESTestCase; @@ -19,14 +19,14 @@ import static java.util.Map.entry; import static org.hamcrest.Matchers.sameInstance; -public class EcsNamespaceProcessorTests extends ESTestCase { +public class NormalizeToOTelProcessorTests extends ESTestCase { - private final EcsNamespaceProcessor processor = new EcsNamespaceProcessor("test", "test processor"); + private final NormalizeToOTelProcessor processor = new NormalizeToOTelProcessor("test", "test processor"); public void testIsOTelDocument_validMinimalOTelDocument() { Map source = new HashMap<>(); source.put("resource", new HashMap<>()); - assertTrue(EcsNamespaceProcessor.isOTelDocument(source)); + assertTrue(NormalizeToOTelProcessor.isOTelDocument(source)); } public void testIsOTelDocument_validOTelDocumentWithScopeAndAttributes() { @@ -34,20 +34,20 @@ public void testIsOTelDocument_validOTelDocumentWithScopeAndAttributes() { source.put("attributes", new HashMap<>()); source.put("resource", new HashMap<>()); source.put("scope", new HashMap<>()); - assertTrue(EcsNamespaceProcessor.isOTelDocument(source)); + assertTrue(NormalizeToOTelProcessor.isOTelDocument(source)); } public void testIsOTelDocument_missingResource() { Map source = new HashMap<>(); source.put("scope", new HashMap<>()); - assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); + assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); } public void testIsOTelDocument_resourceNotMap() { Map source = new HashMap<>(); source.put("resource", "not a map"); source.put("scope", new HashMap<>()); - assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); + assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidResourceAttributes() { @@ -56,14 +56,14 @@ public void testIsOTelDocument_invalidResourceAttributes() { Map source = new HashMap<>(); source.put("resource", resource); source.put("scope", new HashMap<>()); - assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); + assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); } public void testIsOTelDocument_scopeNotMap() { Map source = new HashMap<>(); source.put("resource", new HashMap<>()); source.put("scope", "not a map"); - assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); + assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidAttributes() { @@ -71,7 +71,7 @@ public void testIsOTelDocument_invalidAttributes() { source.put("resource", new HashMap<>()); source.put("scope", new HashMap<>()); source.put("attributes", "not a map"); - assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); + assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidBody() { @@ -79,7 +79,7 @@ public void testIsOTelDocument_invalidBody() { source.put("resource", new HashMap<>()); source.put("scope", new HashMap<>()); source.put("body", "not a map"); - assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); + assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidBodyText() { @@ -89,7 +89,7 @@ public void testIsOTelDocument_invalidBodyText() { source.put("resource", new HashMap<>()); source.put("scope", new HashMap<>()); source.put("body", body); - assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); + assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidBodyStructured() { @@ -99,7 +99,7 @@ public void testIsOTelDocument_invalidBodyStructured() { source.put("resource", new HashMap<>()); source.put("scope", new HashMap<>()); source.put("body", body); - assertFalse(EcsNamespaceProcessor.isOTelDocument(source)); + assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); } public void testIsOTelDocument_validBody() { @@ -110,7 +110,7 @@ public void testIsOTelDocument_validBody() { source.put("resource", new HashMap<>()); source.put("scope", new HashMap<>()); source.put("body", body); - assertTrue(EcsNamespaceProcessor.isOTelDocument(source)); + assertTrue(NormalizeToOTelProcessor.isOTelDocument(source)); } public void testExecute_validOTelDocument() { @@ -212,7 +212,7 @@ public void testRenameSpecialKeys_nestedForm() { source.put("trace", trace); IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - EcsNamespaceProcessor.renameSpecialKeys(document); + NormalizeToOTelProcessor.renameSpecialKeys(document); Map result = document.getSource(); assertEquals("spanIdValue", result.get("span_id")); @@ -231,7 +231,7 @@ public void testRenameSpecialKeys_topLevelDottedField() { source.put("message", "this is a message"); IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - EcsNamespaceProcessor.renameSpecialKeys(document); + NormalizeToOTelProcessor.renameSpecialKeys(document); Map result = document.getSource(); assertEquals("spanIdValue", result.get("span_id")); @@ -254,7 +254,7 @@ public void testRenameSpecialKeys_mixedForm() { source.put("span.id", "topLevelSpanIdValue"); IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - EcsNamespaceProcessor.renameSpecialKeys(document); + NormalizeToOTelProcessor.renameSpecialKeys(document); Map result = document.getSource(); // nested form should take precedence diff --git a/modules/ingest-ecs/src/yamlRestTest/java/org/elasticsearch/ingest/ecs/IngestEcsClientYamlTestSuiteIT.java b/modules/ingest-otel/src/yamlRestTest/java/org/elasticsearch/ingest/otel/IngestOtelClientYamlTestSuiteIT.java similarity index 82% rename from modules/ingest-ecs/src/yamlRestTest/java/org/elasticsearch/ingest/ecs/IngestEcsClientYamlTestSuiteIT.java rename to modules/ingest-otel/src/yamlRestTest/java/org/elasticsearch/ingest/otel/IngestOtelClientYamlTestSuiteIT.java index c93f3ef875c22..85189616c2ea7 100644 --- a/modules/ingest-ecs/src/yamlRestTest/java/org/elasticsearch/ingest/ecs/IngestEcsClientYamlTestSuiteIT.java +++ b/modules/ingest-otel/src/yamlRestTest/java/org/elasticsearch/ingest/otel/IngestOtelClientYamlTestSuiteIT.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.ingest.ecs; +package org.elasticsearch.ingest.otel; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; @@ -17,14 +17,14 @@ import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.junit.ClassRule; -public class IngestEcsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { +public class IngestOtelClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { - public IngestEcsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + public IngestOtelClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @ClassRule - public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("ingest-ecs").build(); + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("ingest-otel").build(); @Override protected String getTestRestCluster() { diff --git a/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespace.yml b/modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_to_otel.yml similarity index 91% rename from modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespace.yml rename to modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_to_otel.yml index beacc600665bc..aaacfb9970a66 100644 --- a/modules/ingest-ecs/src/yamlRestTest/resources/rest-api-spec/test/ingest-ecs/10_ecs_namespace.yml +++ b/modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_to_otel.yml @@ -2,25 +2,25 @@ setup: - do: ingest.put_pipeline: - id: "ecs_namespace_pipeline" + id: "normalize_to_otel_pipeline" body: processors: - - ecs_namespace: {} + - normalize_to_otel: {} --- teardown: - do: ingest.delete_pipeline: - id: "ecs_namespace_pipeline" + id: "normalize_to_otel_pipeline" ignore: 404 --- "Test attributes namespacing": - do: index: - index: ecs_namespacing_test + index: normalize_to_otel_test id: "nested_and_flat_attributes" - pipeline: "ecs_namespace_pipeline" + pipeline: "normalize_to_otel_pipeline" body: { "agent.name": "agentNameValue", "agent": { @@ -76,7 +76,7 @@ teardown: - do: get: - index: ecs_namespacing_test + index: normalize_to_otel_test id: "nested_and_flat_attributes" - match: { _source.resource.attributes.agent\.name: "agentNameValue" } - match: { _source.resource.attributes.agent\.type: "agentTypeValue" } @@ -109,9 +109,9 @@ teardown: "Test rename special keys": - do: index: - index: ecs_namespacing_test + index: normalize_to_otel_test id: "rename_special_keys" - pipeline: "ecs_namespace_pipeline" + pipeline: "normalize_to_otel_pipeline" body: { "span": { "id": "nestedSpanIdValue" @@ -127,7 +127,7 @@ teardown: - do: get: - index: ecs_namespacing_test + index: normalize_to_otel_test id: "rename_special_keys" - match: { _source.span_id: "nestedSpanIdValue" } - match: { _source.severity_text: "topLevelLogLevelValue" } @@ -144,9 +144,9 @@ teardown: "Test valid OTel document": - do: index: - index: ecs_namespacing_test + index: normalize_to_otel_test id: "valid_otel_document" - pipeline: "ecs_namespace_pipeline" + pipeline: "normalize_to_otel_pipeline" body: { "resource": { "attributes": { @@ -171,7 +171,7 @@ teardown: - do: get: - index: ecs_namespacing_test + index: normalize_to_otel_test id: "valid_otel_document" - match: { _source.resource.attributes.foo: "bar" } - match: { _source.scope.foo: "bar" } @@ -187,9 +187,9 @@ teardown: "Test invalid body field": - do: index: - index: ecs_namespacing_test + index: normalize_to_otel_test id: "invalid_body_field" - pipeline: "ecs_namespace_pipeline" + pipeline: "normalize_to_otel_pipeline" body: { "resource": {}, "scope": { @@ -209,7 +209,7 @@ teardown: - do: get: - index: ecs_namespacing_test + index: normalize_to_otel_test id: "invalid_body_field" - match: { _source.attributes.body\.text: 123 } - match: { _source.attributes.body\.structured\.foo: "bar" } From c66663bf037e5a9de59ce56203397324e8ec080d Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Mon, 28 Apr 2025 13:24:08 +0300 Subject: [PATCH 28/45] Apply review comments --- .../ingest/otel/NormalizeToOTelProcessor.java | 24 ++++++++----- .../otel/NormalizeToOTelProcessorTests.java | 34 +++++++++++++++++-- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java index 3ced62d19bbf6..67a9dc92302e2 100644 --- a/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java +++ b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java @@ -114,9 +114,8 @@ public IngestDocument execute(IngestDocument document) { if (keepKey.equals("@timestamp")) { continue; } - Object value = source.remove(keepKey); - if (value != null) { - newAttributes.put(keepKey, value); + if (source.containsKey(keepKey)) { + newAttributes.put(keepKey, source.remove(keepKey)); } } @@ -147,8 +146,8 @@ public IngestDocument execute(IngestDocument document) { } // Flatten attributes - source.replace(ATTRIBUTES_KEY, Maps.flatten(newAttributes, false, false)); - newResource.replace(ATTRIBUTES_KEY, Maps.flatten(newResourceAttributes, false, false)); + source.put(ATTRIBUTES_KEY, Maps.flatten(newAttributes, false, false)); + newResource.put(ATTRIBUTES_KEY, Maps.flatten(newResourceAttributes, false, false)); return document; } @@ -223,9 +222,12 @@ static boolean isOTelDocument(Map source) { */ static void renameSpecialKeys(IngestDocument document) { RENAME_KEYS.forEach((nonOtelName, otelName) -> { + boolean fieldExists = false; + Object value = null; // first look assuming dot notation for nested fields - Object value = document.getFieldValue(nonOtelName, Object.class, true); - if (value != null) { + if (document.hasField(nonOtelName)) { + fieldExists = true; + value = document.getFieldValue(nonOtelName, Object.class, true); document.removeField(nonOtelName); // recursively remove empty parent fields int lastDot = nonOtelName.lastIndexOf('.'); @@ -243,9 +245,13 @@ static void renameSpecialKeys(IngestDocument document) { } } else if (nonOtelName.contains(".")) { // look for dotted field names - value = document.getSource().remove(nonOtelName); + Map source = document.getSource(); + if (source.containsKey(nonOtelName)) { + fieldExists = true; + value = source.remove(nonOtelName); + } } - if (value != null) { + if (fieldExists) { document.setFieldValue(otelName, value); } }); diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java index 19c2199799159..491f451b409ba 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java @@ -17,7 +17,6 @@ import java.util.Map; import static java.util.Map.entry; -import static org.hamcrest.Matchers.sameInstance; public class NormalizeToOTelProcessorTests extends ESTestCase { @@ -121,8 +120,10 @@ public void testExecute_validOTelDocument() { entry("key1", "value1") ); IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); + Map shallowCopy = new HashMap<>(source); processor.execute(document); - assertThat(source, sameInstance(document.getSource())); + // verify that top level keys are not moved when processing a valid OTel document + assertEquals(shallowCopy, document.getSource()); } public void testExecute_nonOTelDocument() { @@ -325,6 +326,35 @@ public void testExecute_moveToAttributeMaps() { assertFalse(source.containsKey("service")); } + public void testKeepNullValues() { + Map source = new HashMap<>(); + Map span = new HashMap<>(); + span.put("id", null); + source.put("span", span); + source.put("log.level", null); + source.put("trace_id", null); + source.put("foo", null); + source.put("agent.name", null); + IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); + + processor.execute(document); + + assertFalse(source.containsKey("span")); + assertTrue(source.containsKey("span_id")); + assertNull(source.get("span_id")); + assertFalse(source.containsKey("log")); + assertTrue(source.containsKey("severity_text")); + assertNull(source.get("severity_text")); + assertFalse(source.containsKey("trace_id")); + Map expectedAttributes = new HashMap<>(); + expectedAttributes.put("foo", null); + expectedAttributes.put("trace_id", null); + assertEquals(expectedAttributes, get(source, "attributes")); + Map expectedResourceAttributes = new HashMap<>(); + expectedResourceAttributes.put("agent.name", null); + assertEquals(expectedResourceAttributes, get(get(source, "resource"), "attributes")); + } + public void testExecute_deepFlattening() { Map source = new HashMap<>(); Map nestedAgent = new HashMap<>(); From 7d536a72b834fcd3f697be4dfd95d4a390a8e903 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Mon, 5 May 2025 15:55:55 +0300 Subject: [PATCH 29/45] Adding accurate resource attributes handling --- .../otel/EcsOTelResourceAttributes.java | 65 ++++ .../ingest/otel/NormalizeToOTelProcessor.java | 57 ++-- .../ingest/otel/EcsFieldsDiscoverer.java | 100 +++++++ .../otel/NormalizeToOTelProcessorTests.java | 163 +++++----- .../ingest/otel/OTelSemConvWebCrawler.java | 281 ++++++++++++++++++ .../ingest/otel/ResourceAttributesTests.java | 80 +++++ .../test/ingest-otel/10_normalize_to_otel.yml | 69 ++--- 7 files changed, 673 insertions(+), 142 deletions(-) create mode 100644 modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/EcsOTelResourceAttributes.java create mode 100644 modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/EcsFieldsDiscoverer.java create mode 100644 modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java create mode 100644 modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java diff --git a/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/EcsOTelResourceAttributes.java b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/EcsOTelResourceAttributes.java new file mode 100644 index 0000000000000..3453a2d55506e --- /dev/null +++ b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/EcsOTelResourceAttributes.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.otel; + +import java.util.Set; + +final class EcsOTelResourceAttributes { + + /** + * The set of ECS (Elastic Common Schema) field names that are mapped to OpenTelemetry resource attributes, + * as defined by the OpenTelemetry Semantic Conventions. + * The list is produced by the {@code ResourceAttributesTests#testAttributesSetUpToDate} test. + * + * @see OpenTelemetry Semantic Conventions + */ + static final Set LATEST = Set.of( + "agent.type", + "agent.build.original", + "agent.name", + "agent.id", + "agent.ephemeral_id", + "agent.version", + "container.image.tag", + "device.model.identifier", + "container.image.hash.all", + "service.node.name", + "process.pid", + "device.id", + "host.mac", + "host.type", + "container.id", + "cloud.availability_zone", + "host.ip", + "container.name", + "container.image.name", + "device.model.name", + "host.name", + "host.id", + "process.executable", + "user_agent.original", + "service.environment", + "cloud.region", + "service.name", + "faas.name", + "device.manufacturer", + "process.args", + "host.architecture", + "cloud.provider", + "container.runtime", + "service.version", + "cloud.service.name", + "cloud.account.id", + "process.command_line", + "faas.version" + ); + + private EcsOTelResourceAttributes() {} +} diff --git a/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java index 67a9dc92302e2..616dfb3958b89 100644 --- a/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java +++ b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.Map; import java.util.Set; @@ -30,8 +31,9 @@ *

More specifically, this processor performs the following operations: *

    *
  • Renames specific ECS fields to their corresponding OpenTelemetry-compatible counterparts.
    • - *
  • Moves fields to the "attributes" and "resource.attributes" namespaces.
    • - *
  • Flattens the "attributes" and "resource.attributes" maps.
    • + *
  • Moves all other fields to the "attributes" namespace.
    • + *
  • Flattens all attributes in the "attributes" namespace.
    • + *
  • Moves resource fields from the "attributes" namespace to the "resource.attributes" namespace.
    • *
* *

If a document is identified as OpenTelemetry-compatible, no transformation is performed. @@ -74,10 +76,6 @@ public class NormalizeToOTelProcessor extends AbstractProcessor { KEEP_KEYS = Set.copyOf(keepKeys); } - private static final String AGENT_PREFIX = "agent"; - private static final String CLOUD_PREFIX = "cloud"; - private static final String HOST_PREFIX = "host"; - private static final String ATTRIBUTES_KEY = "attributes"; private static final String RESOURCE_KEY = "resource"; private static final String SCOPE_KEY = "scope"; @@ -119,35 +117,29 @@ public IngestDocument execute(IngestDocument document) { } } - Map newResource = new HashMap<>(); - Map newResourceAttributes = new HashMap<>(); - newResource.put(ATTRIBUTES_KEY, newResourceAttributes); - source.put(ATTRIBUTES_KEY, newAttributes); - source.put(RESOURCE_KEY, newResource); renameSpecialKeys(document); - // Iterate through all top level keys and move them to the appropriate namespace + // move all top level keys except from specific ones to the "attributes" namespace final var sourceItr = source.entrySet().iterator(); while (sourceItr.hasNext()) { final var entry = sourceItr.next(); - final var key = entry.getKey(); - final var value = entry.getValue(); - if (KEEP_KEYS.contains(key)) { - continue; - } - if (shouldMoveToResourceAttributes(key)) { - newResourceAttributes.put(key, value); - } else { - newAttributes.put(key, value); + if (KEEP_KEYS.contains(entry.getKey()) == false) { + newAttributes.put(entry.getKey(), entry.getValue()); + sourceItr.remove(); } - sourceItr.remove(); } // Flatten attributes - source.put(ATTRIBUTES_KEY, Maps.flatten(newAttributes, false, false)); - newResource.put(ATTRIBUTES_KEY, Maps.flatten(newResourceAttributes, false, false)); + Map flattenAttributes = Maps.flatten(newAttributes, false, false); + source.put(ATTRIBUTES_KEY, flattenAttributes); + + Map newResource = new HashMap<>(); + Map newResourceAttributes = new HashMap<>(); + newResource.put(ATTRIBUTES_KEY, newResourceAttributes); + source.put(RESOURCE_KEY, newResource); + moveResourceAttributes(flattenAttributes, newResourceAttributes); return document; } @@ -257,13 +249,16 @@ static void renameSpecialKeys(IngestDocument document) { }); } - private static boolean shouldMoveToResourceAttributes(String key) { - return key.startsWith(AGENT_PREFIX + ".") - || key.equals(AGENT_PREFIX) - || key.startsWith(CLOUD_PREFIX + ".") - || key.equals(CLOUD_PREFIX) - || key.startsWith(HOST_PREFIX + ".") - || key.equals(HOST_PREFIX); + private static void moveResourceAttributes(Map attributes, Map resourceAttributes) { + Set ecsResourceFields = EcsOTelResourceAttributes.LATEST; + Iterator> attributeIterator = attributes.entrySet().iterator(); + while (attributeIterator.hasNext()) { + Map.Entry entry = attributeIterator.next(); + if (ecsResourceFields.contains(entry.getKey())) { + resourceAttributes.put(entry.getKey(), entry.getValue()); + attributeIterator.remove(); + } + } } public static final class Factory implements Processor.Factory { diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/EcsFieldsDiscoverer.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/EcsFieldsDiscoverer.java new file mode 100644 index 0000000000000..46368de1ba642 --- /dev/null +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/EcsFieldsDiscoverer.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.otel; + +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +class EcsFieldsDiscoverer { + + private static final String ECS_FLAT_FILE_URL = "https://raw.githubusercontent.com/elastic/ecs/main/generated/ecs/ecs_flat.yml"; + private static final String AGENT_FIELDS_PREFIX = "agent."; + + private static final EcsFieldsDiscoverer INSTANCE = new EcsFieldsDiscoverer(); + + private final Map ecsToOTelAttributeNames = new HashMap<>(); + private final Set ecsResourceFields = new HashSet<>(); + + private EcsFieldsDiscoverer() { + try { + collectEcsAttributeNames(); + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Failed to load ECS to OpenTelemetry attribute names", e); + } + } + + Map getEcsToOTelAttributeNames() { + return ecsToOTelAttributeNames; + } + + Set getEcsResourceFields() { + return ecsResourceFields; + } + + static EcsFieldsDiscoverer getInstance() { + return INSTANCE; + } + + private void collectEcsAttributeNames() throws IOException, InterruptedException { + Map ecsFields = loadEcsFields(); + for (Map.Entry entry : ecsFields.entrySet()) { + String ecsName = entry.getKey(); + @SuppressWarnings("unchecked") + Map fieldData = (Map) entry.getValue(); + @SuppressWarnings("unchecked") + List> otelDataEntries = (List>) fieldData.get("otel"); + if (otelDataEntries != null) { + for (Map otelData : otelDataEntries) { + String relation = otelData.get("relation"); + if ("match".equals(relation)) { + ecsToOTelAttributeNames.put(ecsName, ecsName); + } else if ("equivalent".equals(relation)) { + String attribute = otelData.get("attribute"); + if (attribute != null) { + ecsToOTelAttributeNames.put(ecsName, attribute); + } + } + } + } + if (ecsName.startsWith(AGENT_FIELDS_PREFIX)) { + // for now, we consider all agent.* fields as resource attributes, but this may change in the future + ecsResourceFields.add(ecsName); + } + } + } + + private static Map loadEcsFields() throws IOException, InterruptedException { + try (HttpClient httpClient = HttpClient.newHttpClient()) { + HttpRequest request = HttpRequest.newBuilder().uri(URI.create(ECS_FLAT_FILE_URL)).build(); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); + + try ( + InputStream is = response.body(); + XContentParser parser = XContentFactory.xContent(XContentType.YAML).createParser(XContentParserConfiguration.EMPTY, is) + ) { + return parser.map(); + } + } + } +} diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java index 491f451b409ba..a7c0c2cfa6001 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java @@ -262,68 +262,66 @@ public void testRenameSpecialKeys_mixedForm() { assertEquals("nestedSpanIdValue", result.get("span_id")); } - public void testExecute_moveToAttributeMaps() { + public void testExecute_moveFlatAttributes() { Map source = new HashMap<>(); - source.put("agent.name", "agentNameValue"); - Map agent = new HashMap<>(); - agent.put("type", "agentTypeValue"); - source.put("agent", agent); - source.put("cloud.provider", "cloudProviderValue"); - Map cloud = new HashMap<>(); - cloud.put("type", "cloudTypeValue"); - source.put("cloud", cloud); - source.put("host.name", "hostNameValue"); - Map host = new HashMap<>(); - host.put("type", "hostTypeValue"); - source.put("host", host); - source.put("service.name", "serviceNameValue"); - Map service = new HashMap<>(); - service.put("type", "serviceTypeValue"); - source.put("service", service); + Map expectedResourceAttributes = new HashMap<>(); + EcsOTelResourceAttributes.LATEST.forEach(attribute -> { + String value = randomAlphaOfLength(10); + source.put(attribute, value); + expectedResourceAttributes.put(attribute, value); + }); + Map expectedAttributes = Map.of("agent.non-resource", "value", "service.non-resource", "value", "foo", "bar"); + source.putAll(expectedAttributes); IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); processor.execute(document); - Map result = document.getSource(); + assertTrue(source.containsKey("resource")); + Map resource = get(source, "resource"); + Map resourceAttributes = get(resource, "attributes"); + assertEquals(expectedResourceAttributes, resourceAttributes); + EcsOTelResourceAttributes.LATEST.forEach(attribute -> assertFalse(source.containsKey(attribute))); - // all attributes should be flattened - Map expectedResourceAttributes = Map.of( - "agent.name", - "agentNameValue", - "agent.type", - "agentTypeValue", - "cloud.provider", - "cloudProviderValue", - "cloud.type", - "cloudTypeValue", - "host.name", - "hostNameValue", - "host.type", - "hostTypeValue" - ); + assertTrue(source.containsKey("attributes")); + Map attributes = get(source, "attributes"); + assertEquals(expectedAttributes, attributes); + assertFalse(source.containsKey("foo")); + assertFalse(source.containsKey("agent.non-resource")); + assertFalse(source.containsKey("service.non-resource")); + } - assertTrue(result.containsKey("resource")); - Map resource = get(result, "resource"); + public void testExecute_moveNestedAttributes() { + IngestDocument document = new IngestDocument("index", "id", 1, null, null, new HashMap<>()); + + Map expectedResourceAttributes = new HashMap<>(); + EcsOTelResourceAttributes.LATEST.forEach(attribute -> { + String value = randomAlphaOfLength(10); + // parses dots as object notations + document.setFieldValue(attribute, value); + expectedResourceAttributes.put(attribute, value); + }); + Map expectedAttributes = Map.of("agent.non-resource", "value", "service.non-resource", "value", "foo", "bar"); + expectedAttributes.forEach(document::setFieldValue); + + processor.execute(document); + + Map source = document.getSource(); + + assertTrue(source.containsKey("resource")); + Map resource = get(source, "resource"); Map resourceAttributes = get(resource, "attributes"); assertEquals(expectedResourceAttributes, resourceAttributes); - assertNull(resourceAttributes.get("agent")); - assertNull(resourceAttributes.get("cloud")); - assertNull(resourceAttributes.get("host")); - assertFalse(source.containsKey("agent.name")); - assertFalse(source.containsKey("agent")); - assertFalse(source.containsKey("cloud.provider")); - assertFalse(source.containsKey("cloud")); - assertFalse(source.containsKey("host.name")); - assertFalse(source.containsKey("host")); - - Map expectedAttributes = Map.of("service.name", "serviceNameValue", "service.type", "serviceTypeValue"); - - assertTrue(result.containsKey("attributes")); - Map attributes = get(result, "attributes"); + EcsOTelResourceAttributes.LATEST.forEach(attribute -> { + // parse first part of the key + String namespace = attribute.substring(0, attribute.indexOf('.')); + assertFalse(source.containsKey(namespace)); + }); + assertTrue(source.containsKey("attributes")); + Map attributes = get(source, "attributes"); assertEquals(expectedAttributes, attributes); - assertNull(attributes.get("service")); - assertFalse(source.containsKey("service.name")); - assertFalse(source.containsKey("service")); + assertFalse(source.containsKey("foo")); + assertFalse(source.containsKey("agent.non-resource")); + assertFalse(source.containsKey("service.non-resource")); } public void testKeepNullValues() { @@ -357,19 +355,20 @@ public void testKeepNullValues() { public void testExecute_deepFlattening() { Map source = new HashMap<>(); - Map nestedAgent = new HashMap<>(); - nestedAgent.put("name", "agentNameValue"); - Map deeperAgent = new HashMap<>(); - deeperAgent.put("type", "agentTypeValue"); - nestedAgent.put("details", deeperAgent); - source.put("agent", nestedAgent); + Map service = new HashMap<>(); + service.put("name", "serviceNameValue"); + Map node = new HashMap<>(); + node.put("name", "serviceNodeNameValue"); + node.put("type", "serviceNodeTypeValue"); + service.put("node", node); + source.put("service", service); - Map nestedService = new HashMap<>(); - nestedService.put("name", "serviceNameValue"); - Map deeperService = new HashMap<>(); - deeperService.put("type", "serviceTypeValue"); - nestedService.put("details", deeperService); - source.put("service", nestedService); + Map top = new HashMap<>(); + top.put("child", "childValue"); + Map nestedChild = new HashMap<>(); + nestedChild.put("grandchild", "grandchildValue"); + top.put("nested-child", nestedChild); + source.put("top", top); IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); @@ -377,20 +376,32 @@ public void testExecute_deepFlattening() { Map result = document.getSource(); - Map expectedResourceAttributes = Map.of("agent.name", "agentNameValue", "agent.details.type", "agentTypeValue"); + Map expectedResourceAttributes = Map.of( + "service.name", + "serviceNameValue", + "service.node.name", + "serviceNodeNameValue" + ); assertTrue(result.containsKey("resource")); Map resource = get(result, "resource"); Map resourceAttributes = get(resource, "attributes"); assertEquals(expectedResourceAttributes, resourceAttributes); - assertNull(resource.get("agent")); - - Map expectedAttributes = Map.of("service.name", "serviceNameValue", "service.details.type", "serviceTypeValue"); + assertNull(resource.get("service")); + + Map expectedAttributes = Map.of( + "service.node.type", + "serviceNodeTypeValue", + "top.child", + "childValue", + "top.nested-child.grandchild", + "grandchildValue" + ); assertTrue(result.containsKey("attributes")); Map attributes = get(result, "attributes"); assertEquals(expectedAttributes, attributes); - assertNull(attributes.get("service")); + assertNull(attributes.get("top")); } public void testExecute_arraysNotFlattened() { @@ -402,9 +413,8 @@ public void testExecute_arraysNotFlattened() { source.put("agent", nestedAgent); Map nestedService = new HashMap<>(); - nestedService.put("name", "serviceNameValue"); - List serviceArray = List.of("value1", "value2"); - nestedService.put("array", serviceArray); + List serviceNameArray = List.of("value1", "value2"); + nestedService.put("name", serviceNameArray); source.put("service", nestedService); IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); @@ -413,19 +423,18 @@ public void testExecute_arraysNotFlattened() { Map result = document.getSource(); - Map expectedResourceAttributes = Map.of("agent.name", "agentNameValue", "agent.array", agentArray); + Map expectedResourceAttributes = Map.of("agent.name", "agentNameValue", "service.name", serviceNameArray); assertTrue(result.containsKey("resource")); Map resource = get(result, "resource"); Map resourceAttributes = get(resource, "attributes"); assertEquals(expectedResourceAttributes, resourceAttributes); - assertNull(resource.get("agent")); - - Map expectedAttributes = Map.of("service.name", "serviceNameValue", "service.array", serviceArray); assertTrue(result.containsKey("attributes")); Map attributes = get(result, "attributes"); - assertEquals(expectedAttributes, attributes); + assertEquals(Map.of("agent.array", agentArray), attributes); + + assertNull(resource.get("agent")); assertNull(attributes.get("service")); } diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java new file mode 100644 index 0000000000000..dc90f0c647d28 --- /dev/null +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java @@ -0,0 +1,281 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.otel; + +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +class OTelSemConvWebCrawler { + + private static final String OTEL_SEMCONV_ROOT_API_URL = + "https://api.github.com/repos/open-telemetry/semantic-conventions/contents/model?ref=main"; + + private static final String GITHUB_TOKEN = System.getenv("GITHUB_TOKEN"); + + /** + * Relies on GitHub API to crawl the OpenTelemetry semantic conventions repo. + * This method blocks until all resource attributes are collected, which may take a while. + * @return a set of resource attribute names + */ + static Set collectOTelSemConvResourceAttributes() { + Set resourceAttributes = new HashSet<>(); + try ( + // using a relatively high thread pool size, as most tasks it would execute are IO-bound + ExecutorService executor = Executors.newFixedThreadPool(50); + HttpClient httpClient = HttpClient.newBuilder().executor(executor).build(); + Stream yamlFileDownloadUrls = findAllYamlFiles(httpClient) + ) { + // collect all futures without blocking + List>> futures = yamlFileDownloadUrls.map( + url -> extractResourceAttributesFromFile(url, httpClient) + ).toList(); + + // wait for all to complete + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + + // collect results + futures.forEach(future -> resourceAttributes.addAll(future.join())); + } + return resourceAttributes; + } + + /** + * Recursively find all yaml files in the OpenTelemetry semantic-conventions repo. + * The result stream is returned immediately, and the processing is done in separate HttpClient threads. + * @return a stream of yaml file download URLs + */ + private static Stream findAllYamlFiles(HttpClient client) { + + // using a single CompletableFuture that will be completed when all processing is done + CompletableFuture allDone = new CompletableFuture<>(); + + // track pending futures to know when we're done + Set pendingTasks = ConcurrentHashMap.newKeySet(); + + // continuously collect directory URLs as we encounter them + BlockingQueue directoryQueue = new LinkedBlockingQueue<>(); + // add the root directory to the queue + directoryQueue.add(OTEL_SEMCONV_ROOT_API_URL); + + BlockingQueue resultsQueue = new LinkedBlockingQueue<>(); + + // processing is done in a separate thread so that the stream can be returned immediately + try (ExecutorService executor = Executors.newSingleThreadExecutor()) { + CompletableFuture.runAsync(() -> { + try { + while (allDone.isDone() == false) { + // adding task to the pending tasks set to mark that we're processing a directory + Object currentDirectoryProcessingTask = new Object(); + // this prevents race conditions where one thread processes a directory while another thread queries the pending + // tasks before the first thread has added the corresponding http processing tasks to the pending tasks set + pendingTasks.add(currentDirectoryProcessingTask); + try { + String currentUrl = directoryQueue.poll(100, TimeUnit.MILLISECONDS); + + if (currentUrl == null) { + continue; + } + + HttpRequest request = createRequest(currentUrl); + // fork the HTTP request to a separate thread + CompletableFuture> responseFuture = client.sendAsync( + request, + HttpResponse.BodyHandlers.ofInputStream() + ); + + CompletableFuture responseProcessingTask = responseFuture.thenApply(response -> { + try { + if (response.statusCode() == 200) { + try ( + InputStream inputStream = response.body(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(XContentParserConfiguration.EMPTY, inputStream) + ) { + List items = parser.list(); + for (Object item : items) { + if (item instanceof Map entry) { + String type = (String) entry.get("type"); + String name = (String) entry.get("name"); + String downloadUrl = (String) entry.get("download_url"); + String url = (String) entry.get("url"); + + if ("file".equals(type) && (name.endsWith(".yaml") || name.endsWith(".yml"))) { + if (downloadUrl != null) { + resultsQueue.add(downloadUrl); + } + } else if ("dir".equals(type) && url != null) { + directoryQueue.add(url); + } + } + } + } + } else if (response.statusCode() == 403) { + System.err.println( + "GitHub API rate limit exceeded. " + + "Please provide a GitHub token via GITHUB_TOKEN environment variable" + ); + } else { + System.err.println("GitHub API request failed: HTTP " + response.statusCode()); + } + } catch (IOException e) { + System.err.println("Error processing response: " + e.getMessage()); + } + return null; + }); + pendingTasks.add(responseProcessingTask); + + // when this future completes, remove it from pending + responseProcessingTask.whenComplete((result, ex) -> { + pendingTasks.remove(responseProcessingTask); + markDoneIfRequired(pendingTasks, directoryQueue, allDone); + }); + } finally { + // it's now safe to remove the current directory processing task from pending tasks as the + // corresponding response processing tasks, if any, have already been added to the pending tasks set + pendingTasks.remove(currentDirectoryProcessingTask); + markDoneIfRequired(pendingTasks, directoryQueue, allDone); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + allDone.completeExceptionally(e); + throw new RuntimeException(e); + } catch (Exception e) { + allDone.completeExceptionally(e); + throw new RuntimeException(e); + } + }, executor); + } + + // create a custom spliterator that pulls from the queue + Spliterator spliterator = new Spliterators.AbstractSpliterator<>(Long.MAX_VALUE, Spliterator.ORDERED) { + @Override + public boolean tryAdvance(Consumer action) { + try { + String url = resultsQueue.poll(100, TimeUnit.MILLISECONDS); + if (url != null) { + action.accept(url); + return true; + } + return allDone.isDone() == false || resultsQueue.isEmpty() == false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + }; + + // not using parallel stream here because we want to control the concurrency with our own thread pool + return StreamSupport.stream(spliterator, false).onClose(() -> { + // cleanup when the stream is closed + allDone.complete(null); + }); + } + + private static CompletableFuture> extractResourceAttributesFromFile(String fileDownloadUrl, HttpClient httpClient) { + return downloadAndParseYaml(fileDownloadUrl, httpClient).thenApply(yamlData -> { + Set resourceAttributes = new HashSet<>(); + // look for 'groups' entry + Object groupsObj = yamlData.get("groups"); + if (groupsObj instanceof List groups) { + for (Object groupObj : groups) { + if (groupObj instanceof Map groupMap) { + // check if 'type' is 'resource' + if ("resource".equals(groupMap.get("type"))) { + // Get 'attributes' and extract 'ref' fields + Object attributesObj = groupMap.get("attributes"); + if (attributesObj instanceof List attributesList && attributesList.isEmpty() == false) { + for (Object attributeObj : attributesList) { + if (attributeObj instanceof Map attributeMap) { + String refVal = (String) attributeMap.get("ref"); + if (refVal != null) { + resourceAttributes.add(refVal); + } + } + } + } + } + } + } + } + return resourceAttributes; + }); + } + + private static void markDoneIfRequired( + Set pendingTasks, + BlockingQueue directoryQueue, + CompletableFuture allDone + ) { + // There are two types of tasks: directory processing tasks and response processing tasks. + // The only thing that can add to the directory queue is a response processing task, which can only be created + // during the processing of a directory. + // Therefore, if the directory queue is empty and there are no pending tasks - we are done. + if (pendingTasks.isEmpty() && directoryQueue.isEmpty()) { + allDone.complete(null); + } + } + + private static HttpRequest createRequest(String currentUrl) { + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder(URI.create(currentUrl)).header("Accept", "application/vnd.github+json"); + + if (GITHUB_TOKEN != null && GITHUB_TOKEN.isEmpty() == false) { + requestBuilder.header("Authorization", "Bearer " + GITHUB_TOKEN); + } + + return requestBuilder.build(); + } + + private static CompletableFuture> downloadAndParseYaml(String rawFileUrl, HttpClient client) { + HttpRequest request = HttpRequest.newBuilder(URI.create(rawFileUrl)).build(); + CompletableFuture> responseFuture = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()); + return responseFuture.thenApply(response -> { + try ( + InputStream inputStream = response.body(); + XContentParser parser = XContentFactory.xContent(XContentType.YAML) + .createParser(XContentParserConfiguration.EMPTY, inputStream) + ) { + return parser.map(); + } catch (IOException e) { + System.err.println("Error parsing YAML file: " + e.getMessage()); + return Map.of(); + } finally { + if (response.statusCode() != 200) { + System.err.println("GitHub API request failed: HTTP " + response.statusCode()); + } + } + }); + } +} diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java new file mode 100644 index 0000000000000..806409de6e38a --- /dev/null +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.otel; + +import org.elasticsearch.test.ESTestCase; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class ResourceAttributesTests extends ESTestCase { + + public void testResourceAttributesWebCrawler() { + Set resourceAttributes = OTelSemConvWebCrawler.collectOTelSemConvResourceAttributes(); + System.out.println("Resource Attributes: " + resourceAttributes.size()); + for (String attribute : resourceAttributes) { + System.out.println(attribute); + } + } + + public void testEcsToOTelAttributeNames() { + Map attributes = EcsFieldsDiscoverer.getInstance().getEcsToOTelAttributeNames(); + System.out.println("ECS to OTel attribute mappings: " + attributes.size()); + for (Map.Entry entry : attributes.entrySet()) { + System.out.println(entry.getKey() + " --> " + entry.getValue()); + } + } + + public void testAttributesSetUpToDate() { + Map ecsToOTelAttributeNames = EcsFieldsDiscoverer.getInstance().getEcsToOTelAttributeNames(); + Set otelResourceAttributes = OTelSemConvWebCrawler.collectOTelSemConvResourceAttributes(); + Set latestEcsOTelResourceAttributes = new HashSet<>(); + ecsToOTelAttributeNames.forEach((ecsAttributeName, otelAttributeName) -> { + if (otelResourceAttributes.contains(otelAttributeName)) { + latestEcsOTelResourceAttributes.add(ecsAttributeName); + } + }); + latestEcsOTelResourceAttributes.addAll(EcsFieldsDiscoverer.getInstance().getEcsResourceFields()); + if (latestEcsOTelResourceAttributes.equals(EcsOTelResourceAttributes.LATEST)) { + System.out.println("ECS to OTel resource attributes are up to date."); + } else { + System.out.println("ECS to OTel resource attributes are not up to date."); + // find and print the diff + Set addedAttributes = new HashSet<>(latestEcsOTelResourceAttributes); + addedAttributes.removeAll(EcsOTelResourceAttributes.LATEST); + if (addedAttributes.isEmpty() == false) { + System.out.println(); + System.out.println("The current resource attributes set doesn't contain the following attributes:"); + System.out.println("-----------------------------------------------------------------------------"); + for (String attribute : addedAttributes) { + System.out.println(attribute); + } + System.out.println("-----------------------------------------------------------------------------"); + System.out.println(); + } + Set removedAttributes = new HashSet<>(EcsOTelResourceAttributes.LATEST); + removedAttributes.removeAll(latestEcsOTelResourceAttributes); + if (removedAttributes.isEmpty() == false) { + System.out.println(); + System.out.println("The following attributes are no longer considered resource attributes:"); + System.out.println("----------------------------------------------------------------------"); + for (String attribute : removedAttributes) { + System.out.println(attribute); + } + System.out.println("----------------------------------------------------------------------"); + System.out.println(); + } + System.out.println("Consider updating EcsOTelResourceAttributes accordingly"); + System.out.println(); + fail("ECS to OTel resource attributes are not up to date"); + } + } +} diff --git a/modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_to_otel.yml b/modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_to_otel.yml index aaacfb9970a66..118873e4c8656 100644 --- a/modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_to_otel.yml +++ b/modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_to_otel.yml @@ -45,9 +45,22 @@ teardown: "arrayValue2" ] }, - "cloud.provider": "cloudProviderValue", + "cloud.region": "cloudRegionValue", "cloud": { - "type": "cloudTypeValue" + "service": { + "name": [ + "nameArrayValue1", + "nameArrayValue2" + ] + }, + "account.id": [ + { + "key1": "value1" + }, + { + "key2": "value2" + } + ], }, "host.name": "hostNameValue", "host": { @@ -56,21 +69,6 @@ teardown: "service.name": "serviceNameValue", "service": { "type": "serviceTypeValue", - "deep": { - "nested": "nestedValue", - "scalar-array": [ - "arrayValue1", - "arrayValue2" - ], - "object-array": [ - { - "key1": "value1" - }, - { - "key2": "value2" - } - ] - }, } } @@ -80,29 +78,32 @@ teardown: id: "nested_and_flat_attributes" - match: { _source.resource.attributes.agent\.name: "agentNameValue" } - match: { _source.resource.attributes.agent\.type: "agentTypeValue" } - - match: { _source.resource.attributes.agent\.deep\.nested: "nestedValue" } - - match: { _source.resource.attributes.agent\.deep\.scalar-array.0: "arrayValue1" } - - match: { _source.resource.attributes.agent\.deep\.scalar-array.1: "arrayValue2" } - - match: { _source.resource.attributes.agent\.deep\.object-array.0.key1: "value1" } - - match: { _source.resource.attributes.agent\.deep\.object-array.1.key2: "value2" } - - match: { _source.resource.attributes.cloud\.provider: "cloudProviderValue" } - - match: { _source.resource.attributes.cloud\.type: "cloudTypeValue" } + - match: { _source.resource.attributes.cloud\.region: "cloudRegionValue" } + - match: { _source.resource.attributes.cloud\.service\.name: ["nameArrayValue1", "nameArrayValue2"] } + - match: { _source.resource.attributes.cloud\.service\.name.0: "nameArrayValue1" } + - match: { _source.resource.attributes.cloud\.service\.name.1: "nameArrayValue2" } + - match: { _source.resource.attributes.cloud\.account\.id: [{"key1" : "value1"}, {"key2" : "value2"}] } + - match: { _source.resource.attributes.cloud\.account\.id.0.key1: "value1" } + - match: { _source.resource.attributes.cloud\.account\.id.1.key2: "value2" } - match: { _source.resource.attributes.host\.name: "hostNameValue" } - match: { _source.resource.attributes.host\.type: "hostTypeValue" } - - match: { _source.attributes.service\.name: "serviceNameValue" } + - match: { _source.resource.attributes.service\.name: "serviceNameValue" } + - match: { _source.attributes.agent\.scalar-array.0: "arrayValue1" } + - match: { _source.attributes.agent\.scalar-array.1: "arrayValue2" } + - match: { _source.attributes.agent\.deep\.nested: "nestedValue" } + - match: { _source.attributes.agent\.deep\.scalar-array.0: "arrayValue1" } + - match: { _source.attributes.agent\.deep\.scalar-array.1: "arrayValue2" } + - match: { _source.attributes.agent\.deep\.object-array.0.key1: "value1" } + - match: { _source.attributes.agent\.deep\.object-array.1.key2: "value2" } - match: { _source.attributes.service\.type: "serviceTypeValue" } - - match: { _source.attributes.service\.deep\.nested: "nestedValue" } - - match: { _source.attributes.service\.deep\.scalar-array.0: "arrayValue1" } - - match: { _source.attributes.service\.deep\.scalar-array.1: "arrayValue2" } - - match: { _source.attributes.service\.deep\.object-array.0.key1: "value1" } - - match: { _source.attributes.service\.deep\.object-array.1.key2: "value2" } - - match: { _source.agent.name: null } + - match: { _source.agent\.name: null } - match: { _source.agent: null } - - match: { _source.cloud.provider: null } + - match: { _source.agent.type: null } + - match: { _source.cloud\.region: null } - match: { _source.cloud: null } - - match: { _source.host.name: null } + - match: { _source.host\.name: null } - match: { _source.host: null } - - match: { _source.service.name: null } + - match: { _source.service\.name: null } - match: { _source.service: null } --- From bb538a6782e4297a6edc08bae69c4697df513cf8 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Mon, 5 May 2025 17:45:24 +0300 Subject: [PATCH 30/45] Suppress warning for forbidden usage of System.out in tests --- .../ingest/otel/ResourceAttributesTests.java | 65 ++++++++++--------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java index 806409de6e38a..3512172fa4350 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.otel; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.test.ESTestCase; import java.util.HashSet; @@ -17,6 +18,7 @@ public class ResourceAttributesTests extends ESTestCase { + @SuppressForbidden(reason = "Used specifically for the output. Only meant to be run manually, not through CI.") public void testResourceAttributesWebCrawler() { Set resourceAttributes = OTelSemConvWebCrawler.collectOTelSemConvResourceAttributes(); System.out.println("Resource Attributes: " + resourceAttributes.size()); @@ -25,6 +27,7 @@ public void testResourceAttributesWebCrawler() { } } + @SuppressForbidden(reason = "Used specifically for the output. Only meant to be run manually, not through CI.") public void testEcsToOTelAttributeNames() { Map attributes = EcsFieldsDiscoverer.getInstance().getEcsToOTelAttributeNames(); System.out.println("ECS to OTel attribute mappings: " + attributes.size()); @@ -43,38 +46,42 @@ public void testAttributesSetUpToDate() { } }); latestEcsOTelResourceAttributes.addAll(EcsFieldsDiscoverer.getInstance().getEcsResourceFields()); - if (latestEcsOTelResourceAttributes.equals(EcsOTelResourceAttributes.LATEST)) { - System.out.println("ECS to OTel resource attributes are up to date."); - } else { - System.out.println("ECS to OTel resource attributes are not up to date."); - // find and print the diff - Set addedAttributes = new HashSet<>(latestEcsOTelResourceAttributes); - addedAttributes.removeAll(EcsOTelResourceAttributes.LATEST); - if (addedAttributes.isEmpty() == false) { - System.out.println(); - System.out.println("The current resource attributes set doesn't contain the following attributes:"); - System.out.println("-----------------------------------------------------------------------------"); - for (String attribute : addedAttributes) { - System.out.println(attribute); - } - System.out.println("-----------------------------------------------------------------------------"); - System.out.println(); + boolean upToDate = latestEcsOTelResourceAttributes.equals(EcsOTelResourceAttributes.LATEST); + if (upToDate == false) { + printComparisonResults(latestEcsOTelResourceAttributes); + } + assertTrue("ECS-to-OTel resource attributes set is not up to date.", upToDate); + } + + @SuppressForbidden(reason = "Output is used for updating the resource attributes set, not running in the normal PR CI build pipeline") + private static void printComparisonResults(Set latestEcsOTelResourceAttributes) { + // find and print the diff + Set addedAttributes = new HashSet<>(latestEcsOTelResourceAttributes); + addedAttributes.removeAll(EcsOTelResourceAttributes.LATEST); + if (addedAttributes.isEmpty() == false) { + System.out.println(); + System.out.println("The current resource attributes set doesn't contain the following attributes:"); + System.out.println("-----------------------------------------------------------------------------"); + for (String attribute : addedAttributes) { + System.out.println(attribute); } - Set removedAttributes = new HashSet<>(EcsOTelResourceAttributes.LATEST); - removedAttributes.removeAll(latestEcsOTelResourceAttributes); - if (removedAttributes.isEmpty() == false) { - System.out.println(); - System.out.println("The following attributes are no longer considered resource attributes:"); - System.out.println("----------------------------------------------------------------------"); - for (String attribute : removedAttributes) { - System.out.println(attribute); - } - System.out.println("----------------------------------------------------------------------"); - System.out.println(); + System.out.println("-----------------------------------------------------------------------------"); + System.out.println(); + } + Set removedAttributes = new HashSet<>(EcsOTelResourceAttributes.LATEST); + removedAttributes.removeAll(latestEcsOTelResourceAttributes); + if (removedAttributes.isEmpty() == false) { + System.out.println(); + System.out.println("The following attributes are no longer considered resource attributes:"); + System.out.println("----------------------------------------------------------------------"); + for (String attribute : removedAttributes) { + System.out.println(attribute); } - System.out.println("Consider updating EcsOTelResourceAttributes accordingly"); + System.out.println("----------------------------------------------------------------------"); System.out.println(); - fail("ECS to OTel resource attributes are not up to date"); } + System.out.println("Consider updating EcsOTelResourceAttributes accordingly"); + System.out.println(); + fail("ECS to OTel resource attributes are not up to date"); } } From e7c1f9d73a7218149335f95145789acc11a511b1 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Mon, 5 May 2025 18:26:41 +0300 Subject: [PATCH 31/45] Eliminating some more forbidden APIs --- .../ingest/otel/OTelSemConvWebCrawler.java | 45 +++++++++++-------- .../ingest/otel/ResourceAttributesTests.java | 6 ++- .../ConcurrentSeqNoVersioningIT.java | 2 +- .../mvdedupe/MultivalueDedupeTests.java | 2 +- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java index dc90f0c647d28..cef976cbc0ed5 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java @@ -9,6 +9,8 @@ package org.elasticsearch.ingest.otel; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -44,6 +46,8 @@ class OTelSemConvWebCrawler { private static final String GITHUB_TOKEN = System.getenv("GITHUB_TOKEN"); + private static final Logger logger = LogManager.getLogger(OTelSemConvWebCrawler.class); + /** * Relies on GitHub API to crawl the OpenTelemetry semantic conventions repo. * This method blocks until all resource attributes are collected, which may take a while. @@ -142,15 +146,15 @@ private static Stream findAllYamlFiles(HttpClient client) { } } } else if (response.statusCode() == 403) { - System.err.println( + logger.error( "GitHub API rate limit exceeded. " + "Please provide a GitHub token via GITHUB_TOKEN environment variable" ); } else { - System.err.println("GitHub API request failed: HTTP " + response.statusCode()); + logErrorHttpStatusCode(response); } } catch (IOException e) { - System.err.println("Error processing response: " + e.getMessage()); + logger.error("Error processing response: {}", e.getMessage()); } return null; }); @@ -204,6 +208,10 @@ public boolean tryAdvance(Consumer action) { }); } + private static void logErrorHttpStatusCode(HttpResponse response) { + logger.error("GitHub API request failed: HTTP {}", response.statusCode()); + } + private static CompletableFuture> extractResourceAttributesFromFile(String fileDownloadUrl, HttpClient httpClient) { return downloadAndParseYaml(fileDownloadUrl, httpClient).thenApply(yamlData -> { Set resourceAttributes = new HashSet<>(); @@ -261,21 +269,22 @@ private static HttpRequest createRequest(String currentUrl) { private static CompletableFuture> downloadAndParseYaml(String rawFileUrl, HttpClient client) { HttpRequest request = HttpRequest.newBuilder(URI.create(rawFileUrl)).build(); CompletableFuture> responseFuture = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()); - return responseFuture.thenApply(response -> { - try ( - InputStream inputStream = response.body(); - XContentParser parser = XContentFactory.xContent(XContentType.YAML) - .createParser(XContentParserConfiguration.EMPTY, inputStream) - ) { - return parser.map(); - } catch (IOException e) { - System.err.println("Error parsing YAML file: " + e.getMessage()); - return Map.of(); - } finally { - if (response.statusCode() != 200) { - System.err.println("GitHub API request failed: HTTP " + response.statusCode()); - } + return responseFuture.thenApply(OTelSemConvWebCrawler::apply); + } + + private static Map apply(HttpResponse response) { + try ( + InputStream inputStream = response.body(); + XContentParser parser = XContentFactory.xContent(XContentType.YAML).createParser(XContentParserConfiguration.EMPTY, inputStream) + ) { + return parser.map(); + } catch (IOException e) { + logger.error("Error parsing YAML file: {}", e.getMessage()); + return Map.of(); + } finally { + if (response.statusCode() != 200) { + logErrorHttpStatusCode(response); } - }); + } } } diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java index 3512172fa4350..cfa41422e1944 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java @@ -11,11 +11,13 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.test.ESTestCase; +import org.junit.Ignore; import java.util.HashSet; import java.util.Map; import java.util.Set; +//@Ignore public class ResourceAttributesTests extends ESTestCase { @SuppressForbidden(reason = "Used specifically for the output. Only meant to be run manually, not through CI.") @@ -53,7 +55,9 @@ public void testAttributesSetUpToDate() { assertTrue("ECS-to-OTel resource attributes set is not up to date.", upToDate); } - @SuppressForbidden(reason = "Output is used for updating the resource attributes set, not running in the normal PR CI build pipeline") + @SuppressForbidden( + reason = "Output is used for updating the resource attributes set. Running nightly and only prints when not up to date." + ) private static void printComparisonResults(Set latestEcsOTelResourceAttributes) { // find and print the diff Set addedAttributes = new HashSet<>(latestEcsOTelResourceAttributes); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java index 8db81b985b439..3a959dd9cb766 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java @@ -683,7 +683,7 @@ private static LinearizabilityChecker.Event readEvent(StreamInput input) throws @SuppressForbidden(reason = "system err is ok for a command line tool") public static void main(String[] args) throws Exception { if (args.length < 3) { - System.err.println("usage: "); + logger.error("usage: "); } else { runLinearizabilityChecker(new FileInputStream(args[0]), Long.parseLong(args[1]), Long.parseLong(args[2])); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeTests.java index 4caa6415f0cfe..a6e2b0c3fbdba 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeTests.java @@ -465,7 +465,7 @@ private void assertLookup( for (int i = start; i < end; i++) { actualValues.add(valueLookup.apply(lookup.getInt(i) - 1)); } - // System.err.println(actualValues + " " + + // logger.error(actualValues + " " + // v.stream().filter(contained::contains).collect(Collectors.toCollection(TreeSet::new))); assertThat(actualValues, equalTo(v.stream().filter(contained::contains).collect(Collectors.toCollection(TreeSet::new)))); } From 605bdc9121a43876d0cb1a327a6651fcd8ff5ae5 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 5 May 2025 15:35:27 +0000 Subject: [PATCH 32/45] [CI] Auto commit changes from spotless --- .../org/elasticsearch/ingest/otel/ResourceAttributesTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java index cfa41422e1944..6c5d2351ffc9d 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.test.ESTestCase; -import org.junit.Ignore; import java.util.HashSet; import java.util.Map; From e65c0effb0d1f8c2b095c5556ed3e442af9d6670 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 6 May 2025 07:50:14 +0300 Subject: [PATCH 33/45] Reverting shameful refactoring errors --- .../elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java | 2 +- .../compute/operator/mvdedupe/MultivalueDedupeTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java index 3a959dd9cb766..8db81b985b439 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java @@ -683,7 +683,7 @@ private static LinearizabilityChecker.Event readEvent(StreamInput input) throws @SuppressForbidden(reason = "system err is ok for a command line tool") public static void main(String[] args) throws Exception { if (args.length < 3) { - logger.error("usage: "); + System.err.println("usage: "); } else { runLinearizabilityChecker(new FileInputStream(args[0]), Long.parseLong(args[1]), Long.parseLong(args[2])); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeTests.java index a6e2b0c3fbdba..4caa6415f0cfe 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeTests.java @@ -465,7 +465,7 @@ private void assertLookup( for (int i = start; i < end; i++) { actualValues.add(valueLookup.apply(lookup.getInt(i) - 1)); } - // logger.error(actualValues + " " + + // System.err.println(actualValues + " " + // v.stream().filter(contained::contains).collect(Collectors.toCollection(TreeSet::new))); assertThat(actualValues, equalTo(v.stream().filter(contained::contains).collect(Collectors.toCollection(TreeSet::new)))); } From 5c34f94431552e843c94c151db4be9b1990832ae Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 6 May 2025 07:54:19 +0300 Subject: [PATCH 34/45] Disabling ResourceAttributesTests --- .../org/elasticsearch/ingest/otel/ResourceAttributesTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java index 6c5d2351ffc9d..1c1475c1114e2 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java @@ -11,12 +11,13 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.test.ESTestCase; +import org.junit.Ignore; import java.util.HashSet; import java.util.Map; import java.util.Set; -//@Ignore +@Ignore public class ResourceAttributesTests extends ESTestCase { @SuppressForbidden(reason = "Used specifically for the output. Only meant to be run manually, not through CI.") From e11ea5184e9c3958c36f16485e5d1f711220a3a0 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 8 May 2025 15:36:40 +0300 Subject: [PATCH 35/45] Adding local disk crawler --- .../otel/OTelSemConvLocalDiskCrawler.java | 146 ++++++++++++++++++ .../ingest/otel/OTelSemConvWebCrawler.java | 4 +- .../ingest/otel/ResourceAttributesTests.java | 30 +++- 3 files changed, 172 insertions(+), 8 deletions(-) create mode 100644 modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java new file mode 100644 index 0000000000000..3d4a514432171 --- /dev/null +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.otel; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class OTelSemConvLocalDiskCrawler { + + public static final String SEM_CONV_GITHUB_REPO_ZIP_URL = + "https://github.com/open-telemetry/semantic-conventions/archive/refs/heads/main.zip"; + + private static final Logger logger = LogManager.getLogger(OTelSemConvLocalDiskCrawler.class); + + static Set collectOTelSemConvResourceAttributes() { + Path semConvZipFilePath = null; + Path semConvExtractedTmpDirPath = null; + Set resourceAttributes = new HashSet<>(); + try (HttpClient httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.ALWAYS).build()) { + + semConvZipFilePath = Files.createTempFile("otel-semconv-", ".zip"); + + // Download zip + HttpResponse response = httpClient.send( + HttpRequest.newBuilder(URI.create(SEM_CONV_GITHUB_REPO_ZIP_URL)).build(), + HttpResponse.BodyHandlers.ofFile(semConvZipFilePath) + ); + + if (response.statusCode() != 200) { + logger.error("failed to download semantic conventions zip file"); + return resourceAttributes; + } + + // Unzip + semConvExtractedTmpDirPath = Files.createTempDirectory("otel-semconv-extracted-"); + try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(semConvZipFilePath))) { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (entry.isDirectory() == false) { + Path outPath = semConvExtractedTmpDirPath.resolve(entry.getName()); + Files.createDirectories(outPath.getParent()); + Files.copy(zis, outPath, StandardCopyOption.REPLACE_EXISTING); + } + } + } + + // look for the model root at semantic-conventions-main/model + Path semConvModelRootDir = semConvExtractedTmpDirPath.resolve("semantic-conventions-main/model"); + if (Files.exists(semConvModelRootDir) == false) { + logger.error("model directory not found in the extracted zip"); + return resourceAttributes; + } + + try (Stream semConvFileStream = Files.walk(semConvModelRootDir)) { + semConvFileStream.filter(path -> path.toString().endsWith(".yaml") || path.toString().endsWith(".yml")) + .parallel() + .forEach(path -> { + try ( + InputStream inputStream = Files.newInputStream(path); + XContentParser parser = XContentFactory.xContent(XContentType.YAML) + .createParser(XContentParserConfiguration.EMPTY, inputStream) + ) { + Map yamlData = parser.map(); + Object groupsObj = yamlData.get("groups"); + if (groupsObj instanceof List groups) { + for (Object group : groups) { + if (group instanceof Map groupMap && "entity".equals(groupMap.get("type"))) { + Object attrs = groupMap.get("attributes"); + if (attrs instanceof List attrList) { + for (Object attr : attrList) { + if (attr instanceof Map attrMap) { + String refVal = (String) attrMap.get("ref"); + if (refVal != null) { + resourceAttributes.add(refVal); + } + } + } + } + } + } + } + } catch (IOException e) { + logger.error("error parsing yaml file", e); + } + }); + } + } catch (InterruptedException e) { + logger.error("interrupted", e); + } catch (IOException e) { + logger.error("IO exception", e); + } finally { + if (semConvZipFilePath != null) { + try { + Files.deleteIfExists(semConvZipFilePath); + } catch (IOException e) { + logger.warn("failed to delete semconv zip file", e); + } + } + if (semConvExtractedTmpDirPath != null) { + try (Stream semConvFileStream = Files.walk(semConvExtractedTmpDirPath)) { + semConvFileStream.sorted(Comparator.reverseOrder()) // delete files first + .forEach(path -> { + try { + Files.delete(path); + } catch (IOException e) { + logger.warn("failed to delete file: {}", path, e); + } + }); + } catch (IOException e) { + logger.warn("failed to delete semconv zip file", e); + } + } + } + + return resourceAttributes; + } +} diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java index cef976cbc0ed5..2cb45b2b4b188 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java @@ -220,8 +220,8 @@ private static CompletableFuture> extractResourceAttributesFromFile( if (groupsObj instanceof List groups) { for (Object groupObj : groups) { if (groupObj instanceof Map groupMap) { - // check if 'type' is 'resource' - if ("resource".equals(groupMap.get("type"))) { + // check if 'type' is 'entity' + if ("entity".equals(groupMap.get("type"))) { // Get 'attributes' and extract 'ref' fields Object attributesObj = groupMap.get("attributes"); if (attributesObj instanceof List attributesList && attributesList.isEmpty() == false) { diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java index 1c1475c1114e2..0caec8437a87a 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java @@ -9,20 +9,30 @@ package org.elasticsearch.ingest.otel; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.test.ESTestCase; -import org.junit.Ignore; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; -@Ignore +//@LuceneTestCase.Nightly() public class ResourceAttributesTests extends ESTestCase { @SuppressForbidden(reason = "Used specifically for the output. Only meant to be run manually, not through CI.") - public void testResourceAttributesWebCrawler() { - Set resourceAttributes = OTelSemConvWebCrawler.collectOTelSemConvResourceAttributes(); + public void testResourceAttributes_webCrawler() { + testCrawler(OTelSemConvWebCrawler::collectOTelSemConvResourceAttributes); + } + + @SuppressForbidden(reason = "Used specifically for the output. Only meant to be run manually, not through CI.") + public void testResourceAttributes_localDiskCrawler() { + testCrawler(OTelSemConvLocalDiskCrawler::collectOTelSemConvResourceAttributes); + } + + private static void testCrawler(Supplier> otelResourceAttributesSupplier) { + Set resourceAttributes = otelResourceAttributesSupplier.get(); System.out.println("Resource Attributes: " + resourceAttributes.size()); for (String attribute : resourceAttributes) { System.out.println(attribute); @@ -38,9 +48,17 @@ public void testEcsToOTelAttributeNames() { } } - public void testAttributesSetUpToDate() { + public void testAttributesSetUpToDate_localDiskCrawler() { + testAttributesSetUpToDate(OTelSemConvLocalDiskCrawler::collectOTelSemConvResourceAttributes); + } + + public void testAttributesSetUpToDate_webCrawler() { + testAttributesSetUpToDate(OTelSemConvWebCrawler::collectOTelSemConvResourceAttributes); + } + + private static void testAttributesSetUpToDate(Supplier> otelResourceAttributesSupplier) { Map ecsToOTelAttributeNames = EcsFieldsDiscoverer.getInstance().getEcsToOTelAttributeNames(); - Set otelResourceAttributes = OTelSemConvWebCrawler.collectOTelSemConvResourceAttributes(); + Set otelResourceAttributes = otelResourceAttributesSupplier.get(); Set latestEcsOTelResourceAttributes = new HashSet<>(); ecsToOTelAttributeNames.forEach((ecsAttributeName, otelAttributeName) -> { if (otelResourceAttributes.contains(otelAttributeName)) { From c85ddf1b4d6bb2ba39ced59c424b0456fb7824c4 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 8 May 2025 15:37:34 +0300 Subject: [PATCH 36/45] Disabling tests with @LuceneTestCase.Nightly() --- .../org/elasticsearch/ingest/otel/ResourceAttributesTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java index 0caec8437a87a..577b99f168ab2 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java @@ -18,7 +18,7 @@ import java.util.Set; import java.util.function.Supplier; -//@LuceneTestCase.Nightly() +@LuceneTestCase.Nightly() public class ResourceAttributesTests extends ESTestCase { @SuppressForbidden(reason = "Used specifically for the output. Only meant to be run manually, not through CI.") From 717472b7f58f4c58c57d874e9c5b6f06c1f576a4 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 8 May 2025 17:56:12 +0300 Subject: [PATCH 37/45] Disabling forbidden APIs --- .../ingest/otel/OTelSemConvLocalDiskCrawler.java | 2 ++ .../elasticsearch/ingest/otel/ResourceAttributesTests.java | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java index 3d4a514432171..59015ba4a895e 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -41,6 +42,7 @@ public class OTelSemConvLocalDiskCrawler { private static final Logger logger = LogManager.getLogger(OTelSemConvLocalDiskCrawler.class); + @SuppressForbidden(reason = "writing the GitHub repo zip file to the test's runtime temp directory and deleting on exit") static Set collectOTelSemConvResourceAttributes() { Path semConvZipFilePath = null; Path semConvExtractedTmpDirPath = null; diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java index 577b99f168ab2..8a47927c5017b 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.ingest.otel; -import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.test.ESTestCase; @@ -18,19 +17,17 @@ import java.util.Set; import java.util.function.Supplier; -@LuceneTestCase.Nightly() public class ResourceAttributesTests extends ESTestCase { - @SuppressForbidden(reason = "Used specifically for the output. Only meant to be run manually, not through CI.") public void testResourceAttributes_webCrawler() { testCrawler(OTelSemConvWebCrawler::collectOTelSemConvResourceAttributes); } - @SuppressForbidden(reason = "Used specifically for the output. Only meant to be run manually, not through CI.") public void testResourceAttributes_localDiskCrawler() { testCrawler(OTelSemConvLocalDiskCrawler::collectOTelSemConvResourceAttributes); } + @SuppressForbidden(reason = "Used specifically for the output. Only meant to be run manually, not through CI.") private static void testCrawler(Supplier> otelResourceAttributesSupplier) { Set resourceAttributes = otelResourceAttributesSupplier.get(); System.out.println("Resource Attributes: " + resourceAttributes.size()); From d93f0434f792bf3780936d3f759fb32181c7f488 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 8 May 2025 18:04:09 +0300 Subject: [PATCH 38/45] Disabling test and adding javadoc --- ...ests.java => ResourceAttributesTests_disabled.java} | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) rename modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/{ResourceAttributesTests.java => ResourceAttributesTests_disabled.java} (90%) diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests_disabled.java similarity index 90% rename from modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java rename to modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests_disabled.java index 8a47927c5017b..aa071df7260ff 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests_disabled.java @@ -17,7 +17,15 @@ import java.util.Set; import java.util.function.Supplier; -public class ResourceAttributesTests extends ESTestCase { +/** + * DISABLED BY DEFAULT!

+ * These tests are not meant for CI, but rather to be run manually to check whether the static {@link EcsOTelResourceAttributes resource + * attributes set} is up to date with the latest ECS and/or OpenTelemetry Semantic Conventions. + * We may add them to CI in the future, but as such that run periodically (nightly/weekly) and used to notify whenever the resource + * attributes set is not up to date. + */ +@SuppressWarnings("NewClassNamingConvention") +public class ResourceAttributesTests_disabled extends ESTestCase { public void testResourceAttributes_webCrawler() { testCrawler(OTelSemConvWebCrawler::collectOTelSemConvResourceAttributes); From 9d0da6a94d83bd90a26df4a64f6791060043490c Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 8 May 2025 18:37:25 +0300 Subject: [PATCH 39/45] Fix Logger#warn usage --- .../elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java index 59015ba4a895e..90fec54991574 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java @@ -134,7 +134,7 @@ static Set collectOTelSemConvResourceAttributes() { try { Files.delete(path); } catch (IOException e) { - logger.warn("failed to delete file: {}", path, e); + logger.warn("failed to delete file: " + path, e); } }); } catch (IOException e) { From 0a6a5d4e3a645f80a5a8a7ca13f6f39bb1b7b1d6 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Sun, 11 May 2025 17:44:58 +0300 Subject: [PATCH 40/45] Adding documentation --- docs/reference/enrich-processor/index.md | 3 + .../normalize-to-otel-processor.md | 149 ++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 docs/reference/enrich-processor/normalize-to-otel-processor.md diff --git a/docs/reference/enrich-processor/index.md b/docs/reference/enrich-processor/index.md index cf5a21ea27e0e..1df8a481e70d9 100644 --- a/docs/reference/enrich-processor/index.md +++ b/docs/reference/enrich-processor/index.md @@ -84,6 +84,9 @@ Refer to [Enrich your data](docs-content://manage-data/ingest/transform-enrich/d [`network_direction` processor](/reference/enrich-processor/network-direction-processor.md) : Calculates the network direction given a source IP address, destination IP address, and a list of internal networks. +[`normalize_to_otel` processor](/reference/enrich-processor/normalize-to-otel-processor.md) +: Normalizes non-OpenTelemetry documents to be OpenTelemetry-compliant. + [`registered_domain` processor](/reference/enrich-processor/registered-domain-processor.md) : Extracts the registered domain (also known as the effective top-level domain or eTLD), sub-domain, and top-level domain from a fully qualified domain name (FQDN). diff --git a/docs/reference/enrich-processor/normalize-to-otel-processor.md b/docs/reference/enrich-processor/normalize-to-otel-processor.md new file mode 100644 index 0000000000000..7626ea1281edd --- /dev/null +++ b/docs/reference/enrich-processor/normalize-to-otel-processor.md @@ -0,0 +1,149 @@ +--- +navigation_title: "Normalize to OTel" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/normalize-to-otel-processor.html +--- + +# Normalize-to-OTel processor [normalize-to-otel-processor] + + +Detects whether a document is OpenTelemetry-compliant and if not - normalizes it as described below. The resulting document can be queried seamlessly by clients that expect either [ECS](https://www.elastic.co/guide/en/ecs/current/index.html) or OpenTelemetry-[Semantic-Conventions](https://github.com/open-telemetry/semantic-conventions) formats. + +::::{note} +This processor is in tech preview and is not available in our serverless offering. +:::: + +## Detecting OpenTelemetry compliance + +The processor detects OpenTelemetry compliance by checking the following fields: +* `resource` exists as a key and the value is a map +* `resource` either doesn't contain an `attributes` field, or contains an `attributes` field of type map +* `scope` is either missing or a map +* `attributes` is either missing or a map +* `body` is either missing or a map +* `body` either doesn't contain a `text` field, or contains a `text` field of type `String` +* `body` either doesn't contain a `structured` field, or contains a `structured` field that is not of type `String` + +If all of these conditions are met, the document is considered OpenTelemetry-compliant and is not modified by the processor. + +## Normalization + +If the document is not OpenTelemetry-compliant, the processor normalizes it as follows: +* Specific ECS fields are renamed to have their corresponding OpenTelemetry Semantic Conventions attribute names. These include the following: + + | ECS Field | Semantic Conventions Attribute | + |-------------|--------------------------------| + | `span.id` | `span_id` | + | `trace.id` | `trace_id` | + | `message` | `body.text` | + | `log.level` | `severity_text` | + The processor first looks for the nested form of the ECS field and if such does not exist, it looks for a top-level field with the dotted field name. +* Other specific ECS fields that describe resources and have corresponding counterparts in the OpenTelemetry Semantic Conventions are moved to the `resource.attribtues` map. Fields that are considered resource attributes are such that conform to the following conditions: + * They are ECS fields that have corresponding counterparts (either with + the same name or with a different name) in OpenTelemetry Semantic Conventions. + * The corresponding OpenTelemetry attribute is defined in + [Semantic Conventions](https://github.com/open-telemetry/semantic-conventions/tree/main/model) + within a group that is defined as `type: enitity`. +* All other fields, except from `@timestamp` are moved to the `attributes` map. +* All non-array entries of the `attributes` and `resource.attributes` maps are flattened. Flattening means that nested objects are merged into their parent object, and the keys are concatenated with a dot. See examples below. + +## Examples + +If an OpenTelemetry-compliant document is detected, the processor does nothing. For example, the following document will stay unchanged: + +```json +{ + "resource": { + "attributes": { + "service.name": "my-service" + } + }, + "scope": { + "name": "my-library", + "version": "1.0.0" + }, + "attributes": { + "http.method": "GET" + }, + "body": { + "text": "Hello, world!" + } +} +``` + +If a non-OpenTelemetry-compliant document is detected, the processor normalizes it. For example, the following document: + +```json +{ + "@timestamp": "2023-10-01T12:00:00Z", + "service": { + "name": "my-service", + "version": "1.0.0", + "environment": "production", + "language": { + "name": "python", + "version": "3.8" + } + }, + "log": { + "level": "INFO" + }, + "message": "Hello, world!", + "http": { + "method": "GET", + "url": { + "path": "/api/v1/resource" + }, + "headers": [ + { + "name": "Authorization", + "value": "Bearer token" + }, + { + "name": "User-Agent", + "value": "my-client/1.0" + } + ] + }, + "span" : { + "id": "1234567890abcdef" + }, + "span.id": "abcdef1234567890", + "trace.id": "abcdef1234567890abcdef1234567890" +} +``` +will be normalized into the following form: + +```json +{ + "@timestamp": "2023-10-01T12:00:00Z", + "resource": { + "attributes": { + "service.name": "my-service", + "service.version": "1.0.0", + "service.environment": "production" + } + }, + "attributes": { + "service.language.name": "python", + "service.language.version": "3.8", + "http.method": "GET", + "http.url.path": "/api/v1/resource", + "http.headers": [ + { + "name": "Authorization", + "value": "Bearer token" + }, + { + "name": "User-Agent", + "value": "my-client/1.0" + } + ] + }, + "body": { + "text": "Hello, world!" + }, + "span_id": "1234567890abcdef", + "trace_id": "abcdef1234567890abcdef1234567890" +} +``` From cd90ca9a07e6080ef1c5df561723842a837b4e04 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Mon, 12 May 2025 09:38:47 +0300 Subject: [PATCH 41/45] Fix typo --- docs/reference/enrich-processor/normalize-to-otel-processor.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/enrich-processor/normalize-to-otel-processor.md b/docs/reference/enrich-processor/normalize-to-otel-processor.md index 7626ea1281edd..8e5f46d377965 100644 --- a/docs/reference/enrich-processor/normalize-to-otel-processor.md +++ b/docs/reference/enrich-processor/normalize-to-otel-processor.md @@ -44,7 +44,7 @@ If the document is not OpenTelemetry-compliant, the processor normalizes it as f * The corresponding OpenTelemetry attribute is defined in [Semantic Conventions](https://github.com/open-telemetry/semantic-conventions/tree/main/model) within a group that is defined as `type: enitity`. -* All other fields, except from `@timestamp` are moved to the `attributes` map. +* All other fields, except for `@timestamp`, are moved to the `attributes` map. * All non-array entries of the `attributes` and `resource.attributes` maps are flattened. Flattening means that nested objects are merged into their parent object, and the keys are concatenated with a dot. See examples below. ## Examples From a259ce6915ded018bafa482b14232ece49d8c801 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 22 May 2025 11:20:57 +0300 Subject: [PATCH 42/45] Disabling the tests temporarily with @Ignore --- ...utesTests_disabled.java => ResourceAttributesTests.java} | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) rename modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/{ResourceAttributesTests_disabled.java => ResourceAttributesTests.java} (96%) diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests_disabled.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java similarity index 96% rename from modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests_disabled.java rename to modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java index aa071df7260ff..22cdf9d0d2ae8 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests_disabled.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.test.ESTestCase; +import org.junit.Ignore; import java.util.HashSet; import java.util.Map; @@ -24,8 +25,9 @@ * We may add them to CI in the future, but as such that run periodically (nightly/weekly) and used to notify whenever the resource * attributes set is not up to date. */ -@SuppressWarnings("NewClassNamingConvention") -public class ResourceAttributesTests_disabled extends ESTestCase { +@SuppressForbidden(reason = "Disabled temporarily until we set up the periodic CI pipeline") +@Ignore +public class ResourceAttributesTests extends ESTestCase { public void testResourceAttributes_webCrawler() { testCrawler(OTelSemConvWebCrawler::collectOTelSemConvResourceAttributes); From 45f02b693504afd2e7c4f5a1cfe6ebc8eb041c4b Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 22 May 2025 17:26:59 +0300 Subject: [PATCH 43/45] Adding to toc --- docs/reference/enrich-processor/toc.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/enrich-processor/toc.yml b/docs/reference/enrich-processor/toc.yml index f60fe7909d70f..c73e94cf747e1 100644 --- a/docs/reference/enrich-processor/toc.yml +++ b/docs/reference/enrich-processor/toc.yml @@ -28,6 +28,7 @@ toc: - file: kv-processor.md - file: lowercase-processor.md - file: network-direction-processor.md + - file: normalize-to-otel-processor.md - file: pipeline-processor.md - file: redact-processor.md - file: registered-domain-processor.md @@ -44,4 +45,4 @@ toc: - file: uppercase-processor.md - file: urldecode-processor.md - file: uri-parts-processor.md - - file: user-agent-processor.md \ No newline at end of file + - file: user-agent-processor.md From 2b8441df93f94f64b2008390834c9b223ce91005 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 27 May 2025 15:32:20 +0300 Subject: [PATCH 44/45] Remove GitHub-API-based crawler --- ...skCrawler.java => OTelSemConvCrawler.java} | 10 +- .../ingest/otel/OTelSemConvWebCrawler.java | 290 ------------------ .../ingest/otel/ResourceAttributesTests.java | 29 +- 3 files changed, 15 insertions(+), 314 deletions(-) rename modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/{OTelSemConvLocalDiskCrawler.java => OTelSemConvCrawler.java} (94%) delete mode 100644 modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvCrawler.java similarity index 94% rename from modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java rename to modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvCrawler.java index 90fec54991574..d71816b639109 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvLocalDiskCrawler.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvCrawler.java @@ -35,12 +35,18 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -public class OTelSemConvLocalDiskCrawler { +/** + * This class is responsible for crawling and extracting OpenTelemetry semantic convention + * resource attributes from the OpenTelemetry GitHub repository. It handles downloading, + * unzipping, and processing YAML files to extract specific referenced resource attribute names. + * It eventually deletes the downloaded zip file and extracted repository directory. + */ +public class OTelSemConvCrawler { public static final String SEM_CONV_GITHUB_REPO_ZIP_URL = "https://github.com/open-telemetry/semantic-conventions/archive/refs/heads/main.zip"; - private static final Logger logger = LogManager.getLogger(OTelSemConvLocalDiskCrawler.class); + private static final Logger logger = LogManager.getLogger(OTelSemConvCrawler.class); @SuppressForbidden(reason = "writing the GitHub repo zip file to the test's runtime temp directory and deleting on exit") static Set collectOTelSemConvResourceAttributes() { diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java deleted file mode 100644 index 2cb45b2b4b188..0000000000000 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/OTelSemConvWebCrawler.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.ingest.otel; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.XContentType; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -class OTelSemConvWebCrawler { - - private static final String OTEL_SEMCONV_ROOT_API_URL = - "https://api.github.com/repos/open-telemetry/semantic-conventions/contents/model?ref=main"; - - private static final String GITHUB_TOKEN = System.getenv("GITHUB_TOKEN"); - - private static final Logger logger = LogManager.getLogger(OTelSemConvWebCrawler.class); - - /** - * Relies on GitHub API to crawl the OpenTelemetry semantic conventions repo. - * This method blocks until all resource attributes are collected, which may take a while. - * @return a set of resource attribute names - */ - static Set collectOTelSemConvResourceAttributes() { - Set resourceAttributes = new HashSet<>(); - try ( - // using a relatively high thread pool size, as most tasks it would execute are IO-bound - ExecutorService executor = Executors.newFixedThreadPool(50); - HttpClient httpClient = HttpClient.newBuilder().executor(executor).build(); - Stream yamlFileDownloadUrls = findAllYamlFiles(httpClient) - ) { - // collect all futures without blocking - List>> futures = yamlFileDownloadUrls.map( - url -> extractResourceAttributesFromFile(url, httpClient) - ).toList(); - - // wait for all to complete - CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); - - // collect results - futures.forEach(future -> resourceAttributes.addAll(future.join())); - } - return resourceAttributes; - } - - /** - * Recursively find all yaml files in the OpenTelemetry semantic-conventions repo. - * The result stream is returned immediately, and the processing is done in separate HttpClient threads. - * @return a stream of yaml file download URLs - */ - private static Stream findAllYamlFiles(HttpClient client) { - - // using a single CompletableFuture that will be completed when all processing is done - CompletableFuture allDone = new CompletableFuture<>(); - - // track pending futures to know when we're done - Set pendingTasks = ConcurrentHashMap.newKeySet(); - - // continuously collect directory URLs as we encounter them - BlockingQueue directoryQueue = new LinkedBlockingQueue<>(); - // add the root directory to the queue - directoryQueue.add(OTEL_SEMCONV_ROOT_API_URL); - - BlockingQueue resultsQueue = new LinkedBlockingQueue<>(); - - // processing is done in a separate thread so that the stream can be returned immediately - try (ExecutorService executor = Executors.newSingleThreadExecutor()) { - CompletableFuture.runAsync(() -> { - try { - while (allDone.isDone() == false) { - // adding task to the pending tasks set to mark that we're processing a directory - Object currentDirectoryProcessingTask = new Object(); - // this prevents race conditions where one thread processes a directory while another thread queries the pending - // tasks before the first thread has added the corresponding http processing tasks to the pending tasks set - pendingTasks.add(currentDirectoryProcessingTask); - try { - String currentUrl = directoryQueue.poll(100, TimeUnit.MILLISECONDS); - - if (currentUrl == null) { - continue; - } - - HttpRequest request = createRequest(currentUrl); - // fork the HTTP request to a separate thread - CompletableFuture> responseFuture = client.sendAsync( - request, - HttpResponse.BodyHandlers.ofInputStream() - ); - - CompletableFuture responseProcessingTask = responseFuture.thenApply(response -> { - try { - if (response.statusCode() == 200) { - try ( - InputStream inputStream = response.body(); - XContentParser parser = XContentFactory.xContent(XContentType.JSON) - .createParser(XContentParserConfiguration.EMPTY, inputStream) - ) { - List items = parser.list(); - for (Object item : items) { - if (item instanceof Map entry) { - String type = (String) entry.get("type"); - String name = (String) entry.get("name"); - String downloadUrl = (String) entry.get("download_url"); - String url = (String) entry.get("url"); - - if ("file".equals(type) && (name.endsWith(".yaml") || name.endsWith(".yml"))) { - if (downloadUrl != null) { - resultsQueue.add(downloadUrl); - } - } else if ("dir".equals(type) && url != null) { - directoryQueue.add(url); - } - } - } - } - } else if (response.statusCode() == 403) { - logger.error( - "GitHub API rate limit exceeded. " - + "Please provide a GitHub token via GITHUB_TOKEN environment variable" - ); - } else { - logErrorHttpStatusCode(response); - } - } catch (IOException e) { - logger.error("Error processing response: {}", e.getMessage()); - } - return null; - }); - pendingTasks.add(responseProcessingTask); - - // when this future completes, remove it from pending - responseProcessingTask.whenComplete((result, ex) -> { - pendingTasks.remove(responseProcessingTask); - markDoneIfRequired(pendingTasks, directoryQueue, allDone); - }); - } finally { - // it's now safe to remove the current directory processing task from pending tasks as the - // corresponding response processing tasks, if any, have already been added to the pending tasks set - pendingTasks.remove(currentDirectoryProcessingTask); - markDoneIfRequired(pendingTasks, directoryQueue, allDone); - } - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - allDone.completeExceptionally(e); - throw new RuntimeException(e); - } catch (Exception e) { - allDone.completeExceptionally(e); - throw new RuntimeException(e); - } - }, executor); - } - - // create a custom spliterator that pulls from the queue - Spliterator spliterator = new Spliterators.AbstractSpliterator<>(Long.MAX_VALUE, Spliterator.ORDERED) { - @Override - public boolean tryAdvance(Consumer action) { - try { - String url = resultsQueue.poll(100, TimeUnit.MILLISECONDS); - if (url != null) { - action.accept(url); - return true; - } - return allDone.isDone() == false || resultsQueue.isEmpty() == false; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } - } - }; - - // not using parallel stream here because we want to control the concurrency with our own thread pool - return StreamSupport.stream(spliterator, false).onClose(() -> { - // cleanup when the stream is closed - allDone.complete(null); - }); - } - - private static void logErrorHttpStatusCode(HttpResponse response) { - logger.error("GitHub API request failed: HTTP {}", response.statusCode()); - } - - private static CompletableFuture> extractResourceAttributesFromFile(String fileDownloadUrl, HttpClient httpClient) { - return downloadAndParseYaml(fileDownloadUrl, httpClient).thenApply(yamlData -> { - Set resourceAttributes = new HashSet<>(); - // look for 'groups' entry - Object groupsObj = yamlData.get("groups"); - if (groupsObj instanceof List groups) { - for (Object groupObj : groups) { - if (groupObj instanceof Map groupMap) { - // check if 'type' is 'entity' - if ("entity".equals(groupMap.get("type"))) { - // Get 'attributes' and extract 'ref' fields - Object attributesObj = groupMap.get("attributes"); - if (attributesObj instanceof List attributesList && attributesList.isEmpty() == false) { - for (Object attributeObj : attributesList) { - if (attributeObj instanceof Map attributeMap) { - String refVal = (String) attributeMap.get("ref"); - if (refVal != null) { - resourceAttributes.add(refVal); - } - } - } - } - } - } - } - } - return resourceAttributes; - }); - } - - private static void markDoneIfRequired( - Set pendingTasks, - BlockingQueue directoryQueue, - CompletableFuture allDone - ) { - // There are two types of tasks: directory processing tasks and response processing tasks. - // The only thing that can add to the directory queue is a response processing task, which can only be created - // during the processing of a directory. - // Therefore, if the directory queue is empty and there are no pending tasks - we are done. - if (pendingTasks.isEmpty() && directoryQueue.isEmpty()) { - allDone.complete(null); - } - } - - private static HttpRequest createRequest(String currentUrl) { - HttpRequest.Builder requestBuilder = HttpRequest.newBuilder(URI.create(currentUrl)).header("Accept", "application/vnd.github+json"); - - if (GITHUB_TOKEN != null && GITHUB_TOKEN.isEmpty() == false) { - requestBuilder.header("Authorization", "Bearer " + GITHUB_TOKEN); - } - - return requestBuilder.build(); - } - - private static CompletableFuture> downloadAndParseYaml(String rawFileUrl, HttpClient client) { - HttpRequest request = HttpRequest.newBuilder(URI.create(rawFileUrl)).build(); - CompletableFuture> responseFuture = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()); - return responseFuture.thenApply(OTelSemConvWebCrawler::apply); - } - - private static Map apply(HttpResponse response) { - try ( - InputStream inputStream = response.body(); - XContentParser parser = XContentFactory.xContent(XContentType.YAML).createParser(XContentParserConfiguration.EMPTY, inputStream) - ) { - return parser.map(); - } catch (IOException e) { - logger.error("Error parsing YAML file: {}", e.getMessage()); - return Map.of(); - } finally { - if (response.statusCode() != 200) { - logErrorHttpStatusCode(response); - } - } - } -} diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java index 22cdf9d0d2ae8..ce7cde24dfee7 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/ResourceAttributesTests.java @@ -16,7 +16,6 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.function.Supplier; /** * DISABLED BY DEFAULT!

@@ -29,17 +28,9 @@ @Ignore public class ResourceAttributesTests extends ESTestCase { - public void testResourceAttributes_webCrawler() { - testCrawler(OTelSemConvWebCrawler::collectOTelSemConvResourceAttributes); - } - - public void testResourceAttributes_localDiskCrawler() { - testCrawler(OTelSemConvLocalDiskCrawler::collectOTelSemConvResourceAttributes); - } - @SuppressForbidden(reason = "Used specifically for the output. Only meant to be run manually, not through CI.") - private static void testCrawler(Supplier> otelResourceAttributesSupplier) { - Set resourceAttributes = otelResourceAttributesSupplier.get(); + public void testCrawler() { + Set resourceAttributes = OTelSemConvCrawler.collectOTelSemConvResourceAttributes(); System.out.println("Resource Attributes: " + resourceAttributes.size()); for (String attribute : resourceAttributes) { System.out.println(attribute); @@ -55,17 +46,9 @@ public void testEcsToOTelAttributeNames() { } } - public void testAttributesSetUpToDate_localDiskCrawler() { - testAttributesSetUpToDate(OTelSemConvLocalDiskCrawler::collectOTelSemConvResourceAttributes); - } - - public void testAttributesSetUpToDate_webCrawler() { - testAttributesSetUpToDate(OTelSemConvWebCrawler::collectOTelSemConvResourceAttributes); - } - - private static void testAttributesSetUpToDate(Supplier> otelResourceAttributesSupplier) { + public void testAttributesSetUpToDate() { Map ecsToOTelAttributeNames = EcsFieldsDiscoverer.getInstance().getEcsToOTelAttributeNames(); - Set otelResourceAttributes = otelResourceAttributesSupplier.get(); + Set otelResourceAttributes = OTelSemConvCrawler.collectOTelSemConvResourceAttributes(); Set latestEcsOTelResourceAttributes = new HashSet<>(); ecsToOTelAttributeNames.forEach((ecsAttributeName, otelAttributeName) -> { if (otelResourceAttributes.contains(otelAttributeName)) { @@ -76,8 +59,10 @@ private static void testAttributesSetUpToDate(Supplier> otelResource boolean upToDate = latestEcsOTelResourceAttributes.equals(EcsOTelResourceAttributes.LATEST); if (upToDate == false) { printComparisonResults(latestEcsOTelResourceAttributes); + } else { + System.out.println("Latest ECS-to-OTel resource attributes set in EcsOTelResourceAttributes is up to date."); } - assertTrue("ECS-to-OTel resource attributes set is not up to date.", upToDate); + assertTrue("Latest ECS-to-OTel resource attributes set in EcsOTelResourceAttributes is not up to date.", upToDate); } @SuppressForbidden( From f279b7a02189c1098ac5466e28c7a7f2c3591220 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 29 May 2025 14:22:46 +0300 Subject: [PATCH 45/45] Refactor: renaming to normalize_for_stream --- docs/changelog/125699.yaml | 2 +- docs/reference/enrich-processor/index.md | 2 +- ...l-processor.md => normalize-for-stream.md} | 11 ++++--- docs/reference/enrich-processor/toc.yml | 2 +- modules/ingest-otel/build.gradle | 2 +- ...gin.java => NormalizeForStreamPlugin.java} | 4 +-- ....java => NormalizeForStreamProcessor.java} | 8 ++--- ... => NormalizeForStreamProcessorTests.java} | 32 +++++++++---------- ...o_otel.yml => 10_normalize_for_stream.yml} | 30 ++++++++--------- 9 files changed, 48 insertions(+), 45 deletions(-) rename docs/reference/enrich-processor/{normalize-to-otel-processor.md => normalize-for-stream.md} (89%) rename modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/{NormalizeToOTelPlugin.java => NormalizeForStreamPlugin.java} (81%) rename modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/{NormalizeToOTelProcessor.java => NormalizeForStreamProcessor.java} (97%) rename modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/{NormalizeToOTelProcessorTests.java => NormalizeForStreamProcessorTests.java} (93%) rename modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/{10_normalize_to_otel.yml => 10_normalize_for_stream.yml} (90%) diff --git a/docs/changelog/125699.yaml b/docs/changelog/125699.yaml index af3b6d5265b0d..29ee24da4c974 100644 --- a/docs/changelog/125699.yaml +++ b/docs/changelog/125699.yaml @@ -1,5 +1,5 @@ pr: 125699 -summary: Adding `NormalizeToOTelProcessor` +summary: Adding `NormalizeForStreamProcessor` area: Ingest Node type: feature issues: [] diff --git a/docs/reference/enrich-processor/index.md b/docs/reference/enrich-processor/index.md index 1df8a481e70d9..e220e763024e3 100644 --- a/docs/reference/enrich-processor/index.md +++ b/docs/reference/enrich-processor/index.md @@ -84,7 +84,7 @@ Refer to [Enrich your data](docs-content://manage-data/ingest/transform-enrich/d [`network_direction` processor](/reference/enrich-processor/network-direction-processor.md) : Calculates the network direction given a source IP address, destination IP address, and a list of internal networks. -[`normalize_to_otel` processor](/reference/enrich-processor/normalize-to-otel-processor.md) +[`normalize_for_stream` processor](/reference/enrich-processor/normalize-for-stream.md) : Normalizes non-OpenTelemetry documents to be OpenTelemetry-compliant. [`registered_domain` processor](/reference/enrich-processor/registered-domain-processor.md) diff --git a/docs/reference/enrich-processor/normalize-to-otel-processor.md b/docs/reference/enrich-processor/normalize-for-stream.md similarity index 89% rename from docs/reference/enrich-processor/normalize-to-otel-processor.md rename to docs/reference/enrich-processor/normalize-for-stream.md index 8e5f46d377965..0deb8b5d8abc3 100644 --- a/docs/reference/enrich-processor/normalize-to-otel-processor.md +++ b/docs/reference/enrich-processor/normalize-for-stream.md @@ -1,13 +1,16 @@ --- -navigation_title: "Normalize to OTel" +navigation_title: "Normalize for Stream" mapped_pages: - - https://www.elastic.co/guide/en/elasticsearch/reference/current/normalize-to-otel-processor.html + - https://www.elastic.co/guide/en/elasticsearch/reference/current/normalize-for-stream-processor.html --- -# Normalize-to-OTel processor [normalize-to-otel-processor] +# Normalize-for-Stream processor [normalize-for-stream-processor] -Detects whether a document is OpenTelemetry-compliant and if not - normalizes it as described below. The resulting document can be queried seamlessly by clients that expect either [ECS](https://www.elastic.co/guide/en/ecs/current/index.html) or OpenTelemetry-[Semantic-Conventions](https://github.com/open-telemetry/semantic-conventions) formats. +Detects whether a document is OpenTelemetry-compliant and if not - +normalizes it as described below. If used in combination with the OTel-related +mappings such as the ones defined in `logs-otel@template`, the resulting +document can be queried seamlessly by clients that expect either [ECS](https://www.elastic.co/guide/en/ecs/current/index.html) or OpenTelemetry-[Semantic-Conventions](https://github.com/open-telemetry/semantic-conventions) formats. ::::{note} This processor is in tech preview and is not available in our serverless offering. diff --git a/docs/reference/enrich-processor/toc.yml b/docs/reference/enrich-processor/toc.yml index c73e94cf747e1..7da271e6f0554 100644 --- a/docs/reference/enrich-processor/toc.yml +++ b/docs/reference/enrich-processor/toc.yml @@ -28,7 +28,7 @@ toc: - file: kv-processor.md - file: lowercase-processor.md - file: network-direction-processor.md - - file: normalize-to-otel-processor.md + - file: normalize-for-stream.md - file: pipeline-processor.md - file: redact-processor.md - file: registered-domain-processor.md diff --git a/modules/ingest-otel/build.gradle b/modules/ingest-otel/build.gradle index 7818ae32e0eda..54a00508a0a07 100644 --- a/modules/ingest-otel/build.gradle +++ b/modules/ingest-otel/build.gradle @@ -11,7 +11,7 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test' esplugin { description = 'Ingest processor that normalizes ECS documents to OpenTelemetry-compatible namespaces' - classname ='org.elasticsearch.ingest.otel.NormalizeToOTelPlugin' + classname ='org.elasticsearch.ingest.otel.NormalizeForStreamPlugin' } restResources { diff --git a/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelPlugin.java b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeForStreamPlugin.java similarity index 81% rename from modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelPlugin.java rename to modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeForStreamPlugin.java index 205795de435e6..bd88603407ea5 100644 --- a/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelPlugin.java +++ b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeForStreamPlugin.java @@ -15,10 +15,10 @@ import java.util.Map; -public class NormalizeToOTelPlugin extends Plugin implements IngestPlugin { +public class NormalizeForStreamPlugin extends Plugin implements IngestPlugin { @Override public Map getProcessors(Processor.Parameters parameters) { - return Map.of(NormalizeToOTelProcessor.TYPE, new NormalizeToOTelProcessor.Factory()); + return Map.of(NormalizeForStreamProcessor.TYPE, new NormalizeForStreamProcessor.Factory()); } } diff --git a/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeForStreamProcessor.java similarity index 97% rename from modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java rename to modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeForStreamProcessor.java index 616dfb3958b89..d0b2385916823 100644 --- a/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessor.java +++ b/modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeForStreamProcessor.java @@ -39,9 +39,9 @@ *

If a document is identified as OpenTelemetry-compatible, no transformation is performed. * @see org.elasticsearch.ingest.AbstractProcessor */ -public class NormalizeToOTelProcessor extends AbstractProcessor { +public class NormalizeForStreamProcessor extends AbstractProcessor { - public static final String TYPE = "normalize_to_otel"; + public static final String TYPE = "normalize_for_stream"; /** * Mapping of ECS field names to their corresponding OpenTelemetry-compatible counterparts. @@ -83,7 +83,7 @@ public class NormalizeToOTelProcessor extends AbstractProcessor { private static final String TEXT_KEY = "text"; private static final String STRUCTURED_KEY = "structured"; - NormalizeToOTelProcessor(String tag, String description) { + NormalizeForStreamProcessor(String tag, String description) { super(tag, description); } @@ -270,7 +270,7 @@ public Processor create( Map config, ProjectId projectId ) { - return new NormalizeToOTelProcessor(tag, description); + return new NormalizeForStreamProcessor(tag, description); } } } diff --git a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeForStreamProcessorTests.java similarity index 93% rename from modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java rename to modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeForStreamProcessorTests.java index a7c0c2cfa6001..bf2ef92c23dcb 100644 --- a/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeToOTelProcessorTests.java +++ b/modules/ingest-otel/src/test/java/org/elasticsearch/ingest/otel/NormalizeForStreamProcessorTests.java @@ -18,14 +18,14 @@ import static java.util.Map.entry; -public class NormalizeToOTelProcessorTests extends ESTestCase { +public class NormalizeForStreamProcessorTests extends ESTestCase { - private final NormalizeToOTelProcessor processor = new NormalizeToOTelProcessor("test", "test processor"); + private final NormalizeForStreamProcessor processor = new NormalizeForStreamProcessor("test", "test processor"); public void testIsOTelDocument_validMinimalOTelDocument() { Map source = new HashMap<>(); source.put("resource", new HashMap<>()); - assertTrue(NormalizeToOTelProcessor.isOTelDocument(source)); + assertTrue(NormalizeForStreamProcessor.isOTelDocument(source)); } public void testIsOTelDocument_validOTelDocumentWithScopeAndAttributes() { @@ -33,20 +33,20 @@ public void testIsOTelDocument_validOTelDocumentWithScopeAndAttributes() { source.put("attributes", new HashMap<>()); source.put("resource", new HashMap<>()); source.put("scope", new HashMap<>()); - assertTrue(NormalizeToOTelProcessor.isOTelDocument(source)); + assertTrue(NormalizeForStreamProcessor.isOTelDocument(source)); } public void testIsOTelDocument_missingResource() { Map source = new HashMap<>(); source.put("scope", new HashMap<>()); - assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); + assertFalse(NormalizeForStreamProcessor.isOTelDocument(source)); } public void testIsOTelDocument_resourceNotMap() { Map source = new HashMap<>(); source.put("resource", "not a map"); source.put("scope", new HashMap<>()); - assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); + assertFalse(NormalizeForStreamProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidResourceAttributes() { @@ -55,14 +55,14 @@ public void testIsOTelDocument_invalidResourceAttributes() { Map source = new HashMap<>(); source.put("resource", resource); source.put("scope", new HashMap<>()); - assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); + assertFalse(NormalizeForStreamProcessor.isOTelDocument(source)); } public void testIsOTelDocument_scopeNotMap() { Map source = new HashMap<>(); source.put("resource", new HashMap<>()); source.put("scope", "not a map"); - assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); + assertFalse(NormalizeForStreamProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidAttributes() { @@ -70,7 +70,7 @@ public void testIsOTelDocument_invalidAttributes() { source.put("resource", new HashMap<>()); source.put("scope", new HashMap<>()); source.put("attributes", "not a map"); - assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); + assertFalse(NormalizeForStreamProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidBody() { @@ -78,7 +78,7 @@ public void testIsOTelDocument_invalidBody() { source.put("resource", new HashMap<>()); source.put("scope", new HashMap<>()); source.put("body", "not a map"); - assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); + assertFalse(NormalizeForStreamProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidBodyText() { @@ -88,7 +88,7 @@ public void testIsOTelDocument_invalidBodyText() { source.put("resource", new HashMap<>()); source.put("scope", new HashMap<>()); source.put("body", body); - assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); + assertFalse(NormalizeForStreamProcessor.isOTelDocument(source)); } public void testIsOTelDocument_invalidBodyStructured() { @@ -98,7 +98,7 @@ public void testIsOTelDocument_invalidBodyStructured() { source.put("resource", new HashMap<>()); source.put("scope", new HashMap<>()); source.put("body", body); - assertFalse(NormalizeToOTelProcessor.isOTelDocument(source)); + assertFalse(NormalizeForStreamProcessor.isOTelDocument(source)); } public void testIsOTelDocument_validBody() { @@ -109,7 +109,7 @@ public void testIsOTelDocument_validBody() { source.put("resource", new HashMap<>()); source.put("scope", new HashMap<>()); source.put("body", body); - assertTrue(NormalizeToOTelProcessor.isOTelDocument(source)); + assertTrue(NormalizeForStreamProcessor.isOTelDocument(source)); } public void testExecute_validOTelDocument() { @@ -213,7 +213,7 @@ public void testRenameSpecialKeys_nestedForm() { source.put("trace", trace); IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - NormalizeToOTelProcessor.renameSpecialKeys(document); + NormalizeForStreamProcessor.renameSpecialKeys(document); Map result = document.getSource(); assertEquals("spanIdValue", result.get("span_id")); @@ -232,7 +232,7 @@ public void testRenameSpecialKeys_topLevelDottedField() { source.put("message", "this is a message"); IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - NormalizeToOTelProcessor.renameSpecialKeys(document); + NormalizeForStreamProcessor.renameSpecialKeys(document); Map result = document.getSource(); assertEquals("spanIdValue", result.get("span_id")); @@ -255,7 +255,7 @@ public void testRenameSpecialKeys_mixedForm() { source.put("span.id", "topLevelSpanIdValue"); IngestDocument document = new IngestDocument("index", "id", 1, null, null, source); - NormalizeToOTelProcessor.renameSpecialKeys(document); + NormalizeForStreamProcessor.renameSpecialKeys(document); Map result = document.getSource(); // nested form should take precedence diff --git a/modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_to_otel.yml b/modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_for_stream.yml similarity index 90% rename from modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_to_otel.yml rename to modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_for_stream.yml index 118873e4c8656..921c9991c1883 100644 --- a/modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_to_otel.yml +++ b/modules/ingest-otel/src/yamlRestTest/resources/rest-api-spec/test/ingest-otel/10_normalize_for_stream.yml @@ -2,25 +2,25 @@ setup: - do: ingest.put_pipeline: - id: "normalize_to_otel_pipeline" + id: "normalize_for_stream_pipeline" body: processors: - - normalize_to_otel: {} + - normalize_for_stream: {} --- teardown: - do: ingest.delete_pipeline: - id: "normalize_to_otel_pipeline" + id: "normalize_for_stream_pipeline" ignore: 404 --- "Test attributes namespacing": - do: index: - index: normalize_to_otel_test + index: normalize_for_stream_test id: "nested_and_flat_attributes" - pipeline: "normalize_to_otel_pipeline" + pipeline: "normalize_for_stream_pipeline" body: { "agent.name": "agentNameValue", "agent": { @@ -74,7 +74,7 @@ teardown: - do: get: - index: normalize_to_otel_test + index: normalize_for_stream_test id: "nested_and_flat_attributes" - match: { _source.resource.attributes.agent\.name: "agentNameValue" } - match: { _source.resource.attributes.agent\.type: "agentTypeValue" } @@ -110,9 +110,9 @@ teardown: "Test rename special keys": - do: index: - index: normalize_to_otel_test + index: normalize_for_stream_test id: "rename_special_keys" - pipeline: "normalize_to_otel_pipeline" + pipeline: "normalize_for_stream_pipeline" body: { "span": { "id": "nestedSpanIdValue" @@ -128,7 +128,7 @@ teardown: - do: get: - index: normalize_to_otel_test + index: normalize_for_stream_test id: "rename_special_keys" - match: { _source.span_id: "nestedSpanIdValue" } - match: { _source.severity_text: "topLevelLogLevelValue" } @@ -145,9 +145,9 @@ teardown: "Test valid OTel document": - do: index: - index: normalize_to_otel_test + index: normalize_for_stream_test id: "valid_otel_document" - pipeline: "normalize_to_otel_pipeline" + pipeline: "normalize_for_stream_pipeline" body: { "resource": { "attributes": { @@ -172,7 +172,7 @@ teardown: - do: get: - index: normalize_to_otel_test + index: normalize_for_stream_test id: "valid_otel_document" - match: { _source.resource.attributes.foo: "bar" } - match: { _source.scope.foo: "bar" } @@ -188,9 +188,9 @@ teardown: "Test invalid body field": - do: index: - index: normalize_to_otel_test + index: normalize_for_stream_test id: "invalid_body_field" - pipeline: "normalize_to_otel_pipeline" + pipeline: "normalize_for_stream_pipeline" body: { "resource": {}, "scope": { @@ -210,7 +210,7 @@ teardown: - do: get: - index: normalize_to_otel_test + index: normalize_for_stream_test id: "invalid_body_field" - match: { _source.attributes.body\.text: 123 } - match: { _source.attributes.body\.structured\.foo: "bar" }