From 80e15ee3a74b52f6cf9e9eec69d28f04c16cd3c2 Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Tue, 24 Jun 2025 15:07:20 +0200 Subject: [PATCH 1/8] implement spans debug logging --- ...icAutoConfigurationCustomizerProvider.java | 2 + internal-logging/build.gradle.kts | 2 + .../co/elastic/otel/logging/AgentLog.java | 78 ++++++++++++++++++- .../logging/ElasticLoggingCustomizer.java | 2 +- 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/custom/src/main/java/co/elastic/otel/ElasticAutoConfigurationCustomizerProvider.java b/custom/src/main/java/co/elastic/otel/ElasticAutoConfigurationCustomizerProvider.java index 11193333..2d6263e0 100644 --- a/custom/src/main/java/co/elastic/otel/ElasticAutoConfigurationCustomizerProvider.java +++ b/custom/src/main/java/co/elastic/otel/ElasticAutoConfigurationCustomizerProvider.java @@ -22,6 +22,7 @@ import co.elastic.otel.dynamicconfig.BlockableMetricExporter; import co.elastic.otel.dynamicconfig.BlockableSpanExporter; import co.elastic.otel.dynamicconfig.CentralConfig; +import co.elastic.otel.logging.AgentLog; import com.google.auto.service.AutoService; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; @@ -72,6 +73,7 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) { autoConfiguration.addTracerProviderCustomizer( (providerBuilder, properties) -> { CentralConfig.init(providerBuilder, properties); + AgentLog.addSpanLoggingIfRequired(providerBuilder, properties); return providerBuilder; }); } diff --git a/internal-logging/build.gradle.kts b/internal-logging/build.gradle.kts index ba57f0d8..fa707d55 100644 --- a/internal-logging/build.gradle.kts +++ b/internal-logging/build.gradle.kts @@ -10,6 +10,8 @@ dependencies { compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-tooling") compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-bootstrap") + compileOnly("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi") + implementation("io.opentelemetry:opentelemetry-exporter-logging") compileOnly(libs.slf4j.api) implementation(libs.bundles.log4j2) { // Workaround for https://github.com/apache/logging-log4j2/issues/3754 diff --git a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java index 7e3477b8..59cb9df0 100644 --- a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java +++ b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java @@ -18,6 +18,17 @@ */ package co.elastic.otel.logging; +import static java.util.Collections.emptyList; + +import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import java.util.Collection; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; @@ -26,11 +37,21 @@ public class AgentLog { + /** Upstream instrumentation debug boolean option */ + public static final String OTEL_JAVAAGENT_DEBUG = "otel.javaagent.debug"; + private static final String PATTERN = "%d{DEFAULT} [%t] %-5level %logger{36} - %msg{nolookups}%n"; - // logger is an empty string + /** root logger is an empty string */ private static final String ROOT_LOGGER_NAME = ""; + /** + * debug span logging exporter that can be controlled at runtime, only used when logging span + * exporter has not been explicitly configured. + */ + private static final DebugLogSpanExporter debugLogSpanExporter = + new DebugLogSpanExporter(LoggingSpanExporter.create()); + private AgentLog() {} public static void init() { @@ -47,6 +68,22 @@ public static void init() { Configurator.initialize(conf.build(false)); } + public static void addSpanLoggingIfRequired( + SdkTracerProviderBuilder providerBuilder, ConfigProperties config) { + + boolean otelDebug = config.getBoolean(OTEL_JAVAAGENT_DEBUG, false); + + // Replicate behavior of upstream agent: span logging exporter is automatically added + // when not already present when debugging. + // When logging exporter has been explicitly configured, spans logging will be done by the + // explicitly configured logging exporter instance + boolean loggingExporterNotAlreadyConfigured = + !config.getList("otel.traces.exporter", emptyList()).contains("logging"); + if (otelDebug && loggingExporterNotAlreadyConfigured) { + providerBuilder.addSpanProcessor(SimpleSpanProcessor.create(debugLogSpanExporter)); + } + } + public static void setLevel(String level) { switch (level) { case "trace": @@ -87,11 +124,46 @@ public static void setLevel(Level level) { Configurator.setAllLevels(ROOT_LOGGER_NAME, level); - // when debugging we should avoid very chatty http client debug messages + boolean isDebug = level.intLevel() >= Level.DEBUG.intLevel(); + + // When debugging, we should avoid very chatty http client debug messages // this behavior is replicated from the upstream distribution. - if (level.intLevel() >= Level.DEBUG.intLevel()) { + if (isDebug) { Configurator.setLevel("okhttp3.internal.http2", Level.INFO); Configurator.setLevel("okhttp3.internal.concurrent.TaskRunner", Level.INFO); } + + // when debugging the upstream otel agent configures an extra debug exporter + debugLogSpanExporter.setEnabled(isDebug); + } + + private static class DebugLogSpanExporter implements SpanExporter { + + private final SpanExporter delegate; + private final AtomicBoolean enabled; + + DebugLogSpanExporter(SpanExporter delegate) { + this.delegate = delegate; + this.enabled = new AtomicBoolean(false); + } + + void setEnabled(boolean value) { + enabled.set(value); + } + + @Override + public CompletableResultCode export(Collection spans) { + return enabled.get() ? delegate.export(spans) : CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode flush() { + return enabled.get() ? delegate.flush() : CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode shutdown() { + return enabled.get() ? delegate.shutdown() : CompletableResultCode.ofSuccess(); + } } } diff --git a/internal-logging/src/main/java/co/elastic/otel/logging/ElasticLoggingCustomizer.java b/internal-logging/src/main/java/co/elastic/otel/logging/ElasticLoggingCustomizer.java index 93ae61e1..5788f6b2 100644 --- a/internal-logging/src/main/java/co/elastic/otel/logging/ElasticLoggingCustomizer.java +++ b/internal-logging/src/main/java/co/elastic/otel/logging/ElasticLoggingCustomizer.java @@ -46,7 +46,7 @@ public void init(EarlyInitAgentConfig earlyConfig) { AgentLog.init(); Level level = null; - if (earlyConfig.getBoolean("otel.javaagent.debug", false)) { + if (earlyConfig.getBoolean(AgentLog.OTEL_JAVAAGENT_DEBUG, false)) { // set debug logging when enabled through configuration to behave like the upstream // distribution level = Level.DEBUG; From 035764072c12801f2d6d4cbb93d286ba81683cba Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Tue, 24 Jun 2025 18:01:04 +0200 Subject: [PATCH 2/8] fix typo in project dependencies --- internal-logging/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-logging/build.gradle.kts b/internal-logging/build.gradle.kts index fa707d55..da8856bc 100644 --- a/internal-logging/build.gradle.kts +++ b/internal-logging/build.gradle.kts @@ -11,7 +11,7 @@ dependencies { compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-tooling") compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-bootstrap") compileOnly("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi") - implementation("io.opentelemetry:opentelemetry-exporter-logging") + compileOnly("io.opentelemetry:opentelemetry-exporter-logging") compileOnly(libs.slf4j.api) implementation(libs.bundles.log4j2) { // Workaround for https://github.com/apache/logging-log4j2/issues/3754 From f2431068b80e3e773e6ef0af49606cfa7a9d0943 Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Wed, 25 Jun 2025 10:30:26 +0200 Subject: [PATCH 3/8] fix span logging activation --- .../main/java/co/elastic/otel/logging/AgentLog.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java index 59cb9df0..4b1b4fef 100644 --- a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java +++ b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java @@ -71,15 +71,12 @@ public static void init() { public static void addSpanLoggingIfRequired( SdkTracerProviderBuilder providerBuilder, ConfigProperties config) { - boolean otelDebug = config.getBoolean(OTEL_JAVAAGENT_DEBUG, false); - - // Replicate behavior of upstream agent: span logging exporter is automatically added - // when not already present when debugging. - // When logging exporter has been explicitly configured, spans logging will be done by the - // explicitly configured logging exporter instance + // Replicate behavior of the upstream agent: span logging exporter is automatically added when + // not already present when debugging. When logging exporter has been explicitly configured, + // spans logging will be done by the explicitly configured logging exporter instance. boolean loggingExporterNotAlreadyConfigured = !config.getList("otel.traces.exporter", emptyList()).contains("logging"); - if (otelDebug && loggingExporterNotAlreadyConfigured) { + if (loggingExporterNotAlreadyConfigured) { providerBuilder.addSpanProcessor(SimpleSpanProcessor.create(debugLogSpanExporter)); } } From d3aae153144b84776334fddfa9ae4a3162b055af Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Wed, 25 Jun 2025 16:06:52 +0200 Subject: [PATCH 4/8] use json format for spans for elastic logging option --- internal-logging/build.gradle.kts | 1 + .../co/elastic/otel/logging/AgentLog.java | 30 +++++++++++++++---- .../logging/ElasticLoggingCustomizer.java | 18 +++++------ 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/internal-logging/build.gradle.kts b/internal-logging/build.gradle.kts index da8856bc..55ed0d75 100644 --- a/internal-logging/build.gradle.kts +++ b/internal-logging/build.gradle.kts @@ -12,6 +12,7 @@ dependencies { compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-bootstrap") compileOnly("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi") compileOnly("io.opentelemetry:opentelemetry-exporter-logging") + compileOnly("io.opentelemetry:opentelemetry-exporter-logging-otlp") compileOnly(libs.slf4j.api) implementation(libs.bundles.log4j2) { // Workaround for https://github.com/apache/logging-log4j2/issues/3754 diff --git a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java index 4b1b4fef..e72a62e8 100644 --- a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java +++ b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java @@ -21,6 +21,7 @@ import static java.util.Collections.emptyList; import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingSpanExporter; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; @@ -45,16 +46,32 @@ public class AgentLog { /** root logger is an empty string */ private static final String ROOT_LOGGER_NAME = ""; + private static boolean logPlainText = false; + /** * debug span logging exporter that can be controlled at runtime, only used when logging span * exporter has not been explicitly configured. */ - private static final DebugLogSpanExporter debugLogSpanExporter = - new DebugLogSpanExporter(LoggingSpanExporter.create()); + private static DebugLogSpanExporter debugLogSpanExporter = null; private AgentLog() {} - public static void init() { + /** + * Initializes agent logging + * + * @param usePlainTextLog {@literal true} to use plain text logging, `{@literal false} to use JSON + * @param initialLevel initial log level to configure + */ + public static void init(boolean usePlainTextLog, Level initialLevel) { + internalInit(); + setLevel(initialLevel); + logPlainText = usePlainTextLog; + debugLogSpanExporter = + new DebugLogSpanExporter( + logPlainText ? LoggingSpanExporter.create() : OtlpJsonLoggingSpanExporter.create()); + } + + private static void internalInit() { ConfigurationBuilder conf = ConfigurationBuilderFactory.newConfigurationBuilder(); @@ -74,8 +91,11 @@ public static void addSpanLoggingIfRequired( // Replicate behavior of the upstream agent: span logging exporter is automatically added when // not already present when debugging. When logging exporter has been explicitly configured, // spans logging will be done by the explicitly configured logging exporter instance. + + String exporterName = logPlainText ? "logging" : "otlp-logging"; + boolean loggingExporterNotAlreadyConfigured = - !config.getList("otel.traces.exporter", emptyList()).contains("logging"); + !config.getList("otel.traces.exporter", emptyList()).contains(exporterName); if (loggingExporterNotAlreadyConfigured) { providerBuilder.addSpanProcessor(SimpleSpanProcessor.create(debugLogSpanExporter)); } @@ -114,7 +134,7 @@ public static void setLevel(String level) { * * @param level log level */ - public static void setLevel(Level level) { + public static synchronized void setLevel(Level level) { // Using log4j2 implementation allows to change the log level programmatically at runtime // which is not directly possible through the slf4j API and simple implementation used in // upstream distribution. diff --git a/internal-logging/src/main/java/co/elastic/otel/logging/ElasticLoggingCustomizer.java b/internal-logging/src/main/java/co/elastic/otel/logging/ElasticLoggingCustomizer.java index 5788f6b2..59787b68 100644 --- a/internal-logging/src/main/java/co/elastic/otel/logging/ElasticLoggingCustomizer.java +++ b/internal-logging/src/main/java/co/elastic/otel/logging/ElasticLoggingCustomizer.java @@ -22,6 +22,7 @@ import io.opentelemetry.javaagent.bootstrap.InternalLogger; import io.opentelemetry.javaagent.tooling.LoggingCustomizer; import io.opentelemetry.javaagent.tooling.config.EarlyInitAgentConfig; +import java.util.Optional; import org.apache.logging.log4j.Level; import org.slf4j.LoggerFactory; @@ -43,20 +44,19 @@ public void init(EarlyInitAgentConfig earlyConfig) { // make the agent internal logger delegate to slf4j, which will delegate to log4j InternalLogger.initialize(Slf4jInternalLogger::create); - AgentLog.init(); - - Level level = null; - if (earlyConfig.getBoolean(AgentLog.OTEL_JAVAAGENT_DEBUG, false)) { + boolean upstreamDebugEnabled = earlyConfig.getBoolean(AgentLog.OTEL_JAVAAGENT_DEBUG, false); + Level level; + if (upstreamDebugEnabled) { // set debug logging when enabled through configuration to behave like the upstream // distribution level = Level.DEBUG; } else { - String levelConfig = earlyConfig.getString("elastic.otel.javaagent.log.level"); - if (levelConfig != null) { - level = Level.getLevel(levelConfig); - } + level = + Optional.ofNullable(earlyConfig.getString("elastic.otel.javaagent.log.level")) + .map(Level::getLevel) + .orElse(Level.INFO); } - AgentLog.setLevel(level != null ? level : Level.INFO); + AgentLog.init(upstreamDebugEnabled, level); } @Override From 5eb4b3a864056228d26cc942518b832bf05616a1 Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Wed, 25 Jun 2025 18:29:24 +0200 Subject: [PATCH 5/8] fix init ordering --- .../main/java/co/elastic/otel/logging/AgentLog.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java index e72a62e8..217a0956 100644 --- a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java +++ b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java @@ -66,9 +66,6 @@ public static void init(boolean usePlainTextLog, Level initialLevel) { internalInit(); setLevel(initialLevel); logPlainText = usePlainTextLog; - debugLogSpanExporter = - new DebugLogSpanExporter( - logPlainText ? LoggingSpanExporter.create() : OtlpJsonLoggingSpanExporter.create()); } private static void internalInit() { @@ -92,11 +89,16 @@ public static void addSpanLoggingIfRequired( // not already present when debugging. When logging exporter has been explicitly configured, // spans logging will be done by the explicitly configured logging exporter instance. + logPlainText = config.getBoolean(OTEL_JAVAAGENT_DEBUG, false); String exporterName = logPlainText ? "logging" : "otlp-logging"; boolean loggingExporterNotAlreadyConfigured = !config.getList("otel.traces.exporter", emptyList()).contains(exporterName); if (loggingExporterNotAlreadyConfigured) { + debugLogSpanExporter = + new DebugLogSpanExporter( + logPlainText ? LoggingSpanExporter.create() : OtlpJsonLoggingSpanExporter.create()); + providerBuilder.addSpanProcessor(SimpleSpanProcessor.create(debugLogSpanExporter)); } } @@ -151,7 +153,9 @@ public static synchronized void setLevel(Level level) { } // when debugging the upstream otel agent configures an extra debug exporter - debugLogSpanExporter.setEnabled(isDebug); + if(debugLogSpanExporter != null) { + debugLogSpanExporter.setEnabled(isDebug); + } } private static class DebugLogSpanExporter implements SpanExporter { From 723db24b80425bc567a3bb4b531f03781acfafb3 Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Wed, 25 Jun 2025 18:43:55 +0200 Subject: [PATCH 6/8] spotless --- .../src/main/java/co/elastic/otel/logging/AgentLog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java index 217a0956..fe3727d6 100644 --- a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java +++ b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java @@ -153,7 +153,7 @@ public static synchronized void setLevel(Level level) { } // when debugging the upstream otel agent configures an extra debug exporter - if(debugLogSpanExporter != null) { + if (debugLogSpanExporter != null) { debugLogSpanExporter.setEnabled(isDebug); } } From 78aff64d3eefa93b0caa4182583a851213dd7204 Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Fri, 27 Jun 2025 14:47:14 +0200 Subject: [PATCH 7/8] reuse any configured logger --- .../src/main/java/co/elastic/otel/logging/AgentLog.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java index fe3727d6..f185d343 100644 --- a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java +++ b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java @@ -29,6 +29,7 @@ import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import io.opentelemetry.sdk.trace.export.SpanExporter; import java.util.Collection; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; @@ -90,11 +91,11 @@ public static void addSpanLoggingIfRequired( // spans logging will be done by the explicitly configured logging exporter instance. logPlainText = config.getBoolean(OTEL_JAVAAGENT_DEBUG, false); - String exporterName = logPlainText ? "logging" : "otlp-logging"; - boolean loggingExporterNotAlreadyConfigured = - !config.getList("otel.traces.exporter", emptyList()).contains(exporterName); - if (loggingExporterNotAlreadyConfigured) { + List configuredExporters = config.getList("otel.traces.exporter", emptyList()); + boolean loggingConfigured = + configuredExporters.stream().anyMatch(e -> e.equals("logging") || e.equals("otlp-logging")); + if (!loggingConfigured) { debugLogSpanExporter = new DebugLogSpanExporter( logPlainText ? LoggingSpanExporter.create() : OtlpJsonLoggingSpanExporter.create()); From 47764150351a6e6e43f28a8fd96742491c5c85ae Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Fri, 27 Jun 2025 14:48:42 +0200 Subject: [PATCH 8/8] simplify a bit --- .../src/main/java/co/elastic/otel/logging/AgentLog.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java index f185d343..d40c6b16 100644 --- a/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java +++ b/internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java @@ -92,10 +92,8 @@ public static void addSpanLoggingIfRequired( logPlainText = config.getBoolean(OTEL_JAVAAGENT_DEBUG, false); - List configuredExporters = config.getList("otel.traces.exporter", emptyList()); - boolean loggingConfigured = - configuredExporters.stream().anyMatch(e -> e.equals("logging") || e.equals("otlp-logging")); - if (!loggingConfigured) { + List exporters = config.getList("otel.traces.exporter", emptyList()); + if (!exporters.contains("logging") && !exporters.contains("otlp-logging")) { debugLogSpanExporter = new DebugLogSpanExporter( logPlainText ? LoggingSpanExporter.create() : OtlpJsonLoggingSpanExporter.create());