diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java index e0d45db63d4..4ced02dfee5 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java @@ -46,6 +46,7 @@ public class AgentLogExporter implements LogRecordExporter { private final SamplingOverrides exceptionSamplingOverrides; private final LogDataMapper mapper; private final Consumer telemetryItemConsumer; + private final QuickPulse quickPulse; public AgentLogExporter( int severityThreshold, @@ -55,14 +56,12 @@ public AgentLogExporter( @Nullable QuickPulse quickPulse, BatchItemProcessor batchItemProcessor) { this.severityThreshold = severityThreshold; - this.logSamplingOverrides = new SamplingOverrides(logSamplingOverrides); - this.exceptionSamplingOverrides = new SamplingOverrides(exceptionSamplingOverrides); + this.logSamplingOverrides = new SamplingOverrides(logSamplingOverrides, quickPulse); + this.exceptionSamplingOverrides = new SamplingOverrides(exceptionSamplingOverrides, quickPulse); + this.quickPulse = quickPulse; this.mapper = mapper; telemetryItemConsumer = telemetryItem -> { - if (quickPulse != null) { - quickPulse.add(telemetryItem); - } TelemetryObservers.INSTANCE .getObservers() .forEach(consumer -> consumer.accept(telemetryItem)); @@ -120,6 +119,13 @@ private void internalExport(LogRecordData log) { SpanContext spanContext = log.getSpanContext(); Double parentSpanSampleRate = log.getAttributes().get(AiSemanticAttributes.SAMPLE_RATE); + TelemetryItem telemetryItem = null; + if (quickPulse != null && quickPulse.isEnabled()) { + telemetryItem = mapper.map(log, stack, parentSpanSampleRate); + logger.debug("adding log to quick pulse: {}", telemetryItem.toJsonString()); + quickPulse.add(telemetryItem); + } + AiFixedPercentageSampler sampler = samplingOverrides.getOverride(log.getAttributes()); boolean hasSamplingOverride = sampler != null; @@ -136,6 +142,7 @@ private void internalExport(LogRecordData log) { if (hasSamplingOverride) { SamplingResult samplingResult = sampler.shouldSampleLog(spanContext, parentSpanSampleRate); if (samplingResult.getDecision() != SamplingDecision.RECORD_AND_SAMPLE) { + logger.info("Sampling out log for Breeze: {}", log.getBodyValue().asString()); return; } sampleRate = samplingResult.getAttributes().get(AiSemanticAttributes.SAMPLE_RATE); @@ -148,7 +155,9 @@ private void internalExport(LogRecordData log) { logger.debug("exporting log: {}", log); // TODO (trask) no longer need to check AiSemanticAttributes.SAMPLE_RATE in map() method - TelemetryItem telemetryItem = mapper.map(log, stack, sampleRate); + if (telemetryItem == null) { + telemetryItem = mapper.map(log, stack, sampleRate); + } telemetryItemConsumer.accept(telemetryItem); exportingLogLogger.recordSuccess(); diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentSpanExporter.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentSpanExporter.java index 8afa59ed925..d9f8b4b6844 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentSpanExporter.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentSpanExporter.java @@ -8,7 +8,6 @@ import com.azure.monitor.opentelemetry.autoconfigure.implementation.SpanDataMapper; import com.azure.monitor.opentelemetry.autoconfigure.implementation.logging.OperationLogger; import com.azure.monitor.opentelemetry.autoconfigure.implementation.models.TelemetryItem; -import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.QuickPulse; import com.azure.monitor.opentelemetry.autoconfigure.implementation.utils.Strings; import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; @@ -18,7 +17,6 @@ import io.opentelemetry.sdk.trace.export.SpanExporter; import java.util.Collection; import java.util.function.Consumer; -import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,14 +32,10 @@ public final class AgentSpanExporter implements SpanExporter { public AgentSpanExporter( SpanDataMapper mapper, - @Nullable QuickPulse quickPulse, BatchItemProcessor batchItemProcessor) { this.mapper = mapper; telemetryItemConsumer = telemetryItem -> { - if (quickPulse != null) { - quickPulse.add(telemetryItem); - } TelemetryObservers.INSTANCE .getObservers() .forEach(consumer -> consumer.accept(telemetryItem)); diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RuntimeConfigurator.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RuntimeConfigurator.java index 5f41773edf0..3a8a3971e0f 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RuntimeConfigurator.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RuntimeConfigurator.java @@ -8,6 +8,7 @@ import ch.qos.logback.classic.LoggerContext; import com.azure.monitor.opentelemetry.autoconfigure.implementation.heartbeat.HeartbeatExporter; import com.azure.monitor.opentelemetry.autoconfigure.implementation.models.TelemetryItem; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.QuickPulse; import com.azure.monitor.opentelemetry.autoconfigure.implementation.utils.Strings; import com.microsoft.applicationinsights.agent.internal.classicsdk.BytecodeUtilImpl; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; @@ -137,7 +138,7 @@ public void apply(RuntimeConfiguration runtimeConfig) { || !Objects.equals(runtimeConfig.sampling.percentage, currentConfig.sampling.percentage) || !Objects.equals( runtimeConfig.sampling.requestsPerSecond, currentConfig.sampling.requestsPerSecond)) { - updateSampling(enabled, runtimeConfig.sampling, runtimeConfig.samplingPreview); + updateSampling(enabled, runtimeConfig.sampling, runtimeConfig.samplingPreview, telemetryClient.getQuickPulse()); } // initialize Profiler @@ -198,7 +199,8 @@ static void updatePropagation( static void updateSampling( boolean enabled, Configuration.Sampling sampling, - Configuration.SamplingPreview samplingPreview) { + Configuration.SamplingPreview samplingPreview, + QuickPulse quickPulse) { if (!enabled) { DelegatingSampler.getInstance().reset(); @@ -206,7 +208,8 @@ static void updateSampling( return; } - DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(sampling, samplingPreview)); + DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(sampling, samplingPreview, quickPulse)); + // call setQuickPulse method here in delegate if (sampling.percentage != null) { BytecodeUtilImpl.samplingPercentage = sampling.percentage.floatValue(); } else { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java index f2a0864f40f..fe22871bb85 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java @@ -11,6 +11,7 @@ import com.azure.monitor.opentelemetry.autoconfigure.implementation.AzureMonitorLogRecordExporterProvider; import com.azure.monitor.opentelemetry.autoconfigure.implementation.AzureMonitorMetricExporterProvider; import com.azure.monitor.opentelemetry.autoconfigure.implementation.AzureMonitorSpanExporterProvider; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.LiveMetricsSpanProcessor; import com.azure.monitor.opentelemetry.autoconfigure.implementation.LogDataMapper; import com.azure.monitor.opentelemetry.autoconfigure.implementation.MetricDataMapper; import com.azure.monitor.opentelemetry.autoconfigure.implementation.SpanDataMapper; @@ -166,6 +167,28 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) { TelemetryClient.setActive(telemetryClient); + QuickPulse quickPulse; + if (configuration.preview.liveMetrics.enabled) { + quickPulse = + QuickPulse.create( + LazyHttpClient.newHttpPipeLineWithDefaultRedirect(configuration.authentication), + () -> { + ConnectionString connectionString = telemetryClient.getConnectionString(); + return connectionString == null ? null : connectionString.getLiveEndpoint(); + }, + telemetryClient::getInstrumentationKey, + telemetryClient.getRoleName(), + telemetryClient.getRoleInstance(), + FirstEntryPoint.getAgentVersion()); + } else { + quickPulse = null; + } + // quickPulse needs to be set before the runtimeConfigurator is created, so that when + // the telemetry client is passed to the runtime configurator, the runtime configurator can + // use it to pass quickPulse to the RuntimeConfigurator.updateSampling method. Sampling may + // use quickPulse.isEnabled to determine if telemetry should be dropped or record only. + telemetryClient.setQuickPulse(quickPulse); + // TODO (heya) remove duplicate code in both RuntimeConfigurator and SecondEntryPoint RuntimeConfigurator runtimeConfigurator = new RuntimeConfigurator( @@ -220,26 +243,15 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) { // TODO (trask) add this method to AutoConfigurationCustomizer upstream? ((AutoConfiguredOpenTelemetrySdkBuilder) autoConfiguration).disableShutdownHook(); - QuickPulse quickPulse; - if (configuration.preview.liveMetrics.enabled) { - quickPulse = - QuickPulse.create( - LazyHttpClient.newHttpPipeLineWithDefaultRedirect(configuration.authentication), - () -> { - ConnectionString connectionString = telemetryClient.getConnectionString(); - return connectionString == null ? null : connectionString.getLiveEndpoint(); - }, - telemetryClient::getInstrumentationKey, - telemetryClient.getRoleName(), - telemetryClient.getRoleInstance(), - FirstEntryPoint.getAgentVersion()); - } else { - quickPulse = null; - } - telemetryClient.setQuickPulse(quickPulse); - AtomicBoolean firstLogRecordProcessor = new AtomicBoolean(true); + + List exceptionSamplingOverrides = + configuration.preview.sampling.overrides.stream() + .filter(override -> override.telemetryType == SamplingTelemetryType.EXCEPTION) + .collect(Collectors.toList()); + SpanDataMapper mapper = createSpanDataMapper(telemetryClient, configuration.preview.captureHttpServer4xxAsError, new SamplingOverrides(exceptionSamplingOverrides, quickPulse)); + autoConfiguration .addPropertiesSupplier( () -> { @@ -279,7 +291,7 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) { .addSpanExporterCustomizer( (spanExporter, configProperties) -> { if (spanExporter instanceof AzureMonitorSpanExporterProvider.MarkerSpanExporter) { - return buildTraceExporter(configuration, telemetryClient, quickPulse); + return buildTraceExporter(configuration, telemetryClient, mapper); } return wrapSpanExporter(spanExporter, configuration); }) @@ -302,7 +314,7 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) { } }) .addTracerProviderCustomizer( - (builder, otelConfig) -> configureTracing(builder, configuration)) + (builder, otelConfig) -> configureTracing(builder, configuration, quickPulse, mapper)) .addMeterProviderCustomizer( (builder, otelConfig) -> configureMetrics(builder, configuration)); @@ -320,17 +332,12 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) { } private static SpanExporter buildTraceExporter( - Configuration configuration, TelemetryClient telemetryClient, QuickPulse quickPulse) { - List exceptionSamplingOverrides = - configuration.preview.sampling.overrides.stream() - .filter(override -> override.telemetryType == SamplingTelemetryType.EXCEPTION) - .collect(Collectors.toList()); + Configuration configuration, TelemetryClient telemetryClient, SpanDataMapper mapper) { + startupLogger.info("calling createSpanExporter"); SpanExporter spanExporter = createSpanExporter( telemetryClient, - quickPulse, - configuration.preview.captureHttpServer4xxAsError, - new SamplingOverrides(exceptionSamplingOverrides)); + mapper); return wrapSpanExporter(spanExporter, configuration); } @@ -519,7 +526,7 @@ private static Set initStatsbeatFeatureSet(Configuration config) { } private static SdkTracerProviderBuilder configureTracing( - SdkTracerProviderBuilder tracerProvider, Configuration configuration) { + SdkTracerProviderBuilder tracerProvider, Configuration configuration, QuickPulse quickPulse, SpanDataMapper mapper) { boolean enabled = !Strings.isNullOrEmpty(configuration.connectionString); RuntimeConfigurator.updatePropagation( @@ -527,7 +534,7 @@ private static SdkTracerProviderBuilder configureTracing( configuration.preview.additionalPropagators, configuration.preview.legacyRequestIdPropagation.enabled); RuntimeConfigurator.updateSampling( - enabled, configuration.sampling, configuration.preview.sampling); + enabled, configuration.sampling, configuration.preview.sampling, quickPulse); tracerProvider.addSpanProcessor(new AzureMonitorSpanProcessor()); if (!configuration.preview.inheritedAttributes.isEmpty()) { @@ -551,51 +558,60 @@ private static SdkTracerProviderBuilder configureTracing( tracerProvider.addSpanProcessor(new AiLegacyHeaderSpanProcessor()); } + if (quickPulse != null) { + tracerProvider.addSpanProcessor(new LiveMetricsSpanProcessor(quickPulse, mapper)); + } + return tracerProvider; } - private static SpanExporter createSpanExporter( + private static SpanDataMapper createSpanDataMapper( TelemetryClient telemetryClient, - @Nullable QuickPulse quickPulse, boolean captureHttpServer4xxAsError, SamplingOverrides exceptionSamplingOverrides) { + return new SpanDataMapper( + captureHttpServer4xxAsError, + telemetryClient::populateDefaults, + (event, instrumentationName) -> { + boolean lettuce51 = instrumentationName.equals("io.opentelemetry.lettuce-5.1"); + if (lettuce51 && event.getName().startsWith("redis.encode.")) { + // special case as these are noisy and come from the underlying library itself + return true; + } + boolean grpc16 = instrumentationName.equals("io.opentelemetry.grpc-1.6"); + if (grpc16 && event.getName().equals("message")) { + // OpenTelemetry semantic conventions define semi-noisy grpc events + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/rpc.md#events + // + // we want to suppress these (at least by default) + return true; + } + return false; + }, + (span, event) -> { + AiFixedPercentageSampler sampler = + exceptionSamplingOverrides.getOverride(event.getAttributes()); + startupLogger.info("Calling span sampling function with span id {}, event name {}, is sampler null {}, sampler decision {}", + span.getSpanContext().getSpanId(), event.getName(), sampler != null); + return sampler != null + && sampler + .shouldSampleLog( + span.getSpanContext(), + span.getAttributes().get(AiSemanticAttributes.SAMPLE_RATE)) + .getDecision() + == SamplingDecision.DROP; + }); + } - SpanDataMapper mapper = - new SpanDataMapper( - captureHttpServer4xxAsError, - telemetryClient::populateDefaults, - (event, instrumentationName) -> { - boolean lettuce51 = instrumentationName.equals("io.opentelemetry.lettuce-5.1"); - if (lettuce51 && event.getName().startsWith("redis.encode.")) { - // special case as these are noisy and come from the underlying library itself - return true; - } - boolean grpc16 = instrumentationName.equals("io.opentelemetry.grpc-1.6"); - if (grpc16 && event.getName().equals("message")) { - // OpenTelemetry semantic conventions define semi-noisy grpc events - // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/rpc.md#events - // - // we want to suppress these (at least by default) - return true; - } - return false; - }, - (span, event) -> { - AiFixedPercentageSampler sampler = - exceptionSamplingOverrides.getOverride(event.getAttributes()); - return sampler != null - && sampler - .shouldSampleLog( - span.getSpanContext(), - span.getAttributes().get(AiSemanticAttributes.SAMPLE_RATE)) - .getDecision() - == SamplingDecision.DROP; - }); - BatchItemProcessor batchItemProcessor = telemetryClient.getGeneralBatchItemProcessor(); + private static SpanExporter createSpanExporter( + TelemetryClient telemetryClient, + SpanDataMapper mapper) { + BatchItemProcessor batchItemProcessor = telemetryClient.getGeneralBatchItemProcessor(); + startupLogger.info("Create agentSpanExporter for statsbeat"); return new StatsbeatSpanExporter( - new AgentSpanExporter(mapper, quickPulse, batchItemProcessor), + new AgentSpanExporter(mapper, batchItemProcessor), telemetryClient.getStatsbeatModule()); } @@ -611,9 +627,11 @@ private static SpanExporter wrapSpanExporter( for (ProcessorConfig processorConfig : processorConfigs) { switch (processorConfig.type) { case ATTRIBUTE: + startupLogger.info("Adding attribute processor to span exporter"); spanExporter = new SpanExporterWithAttributeProcessor(processorConfig, spanExporter); break; case SPAN: + startupLogger.info("Adding span processor to span exporter"); spanExporter = new ExporterWithSpanProcessor(processorConfig, spanExporter); break; default: diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiFixedPercentageSampler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiFixedPercentageSampler.java index d9528d010f6..0be7f9a1ba7 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiFixedPercentageSampler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiFixedPercentageSampler.java @@ -3,7 +3,9 @@ package com.microsoft.applicationinsights.agent.internal.sampling; +import com.azure.core.util.logging.ClientLogger; import com.azure.monitor.opentelemetry.autoconfigure.implementation.AiSemanticAttributes; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.QuickPulse; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; @@ -20,12 +22,17 @@ public class AiFixedPercentageSampler implements Sampler { private final double percentage; - public static AiFixedPercentageSampler create(double percentage) { - return new AiFixedPercentageSampler(percentage); + private static final ClientLogger logger = new ClientLogger(AiFixedPercentageSampler.class); + + private final QuickPulse quickPulse; + + public static AiFixedPercentageSampler create(double percentage, QuickPulse quickPulse) { + return new AiFixedPercentageSampler(percentage, quickPulse); } - private AiFixedPercentageSampler(double percentage) { + private AiFixedPercentageSampler(double percentage, QuickPulse quickPulse) { this.percentage = percentage; + this.quickPulse = quickPulse; } @Override @@ -37,6 +44,12 @@ public SamplingResult shouldSample( Attributes attributes, List parentLinks) { + String spanId = ""; + if (!parentLinks.isEmpty()) { + spanId = parentLinks.get(0).getSpanContext().getSpanId(); + } + logger.info("shouldSample called with traceId {}, name {}, a span id {}", + traceId, name, spanId); Span parentSpan = Span.fromContext(parentContext); SpanContext parentSpanContext = parentSpan.getSpanContext(); Double parentSpanSampleRate = null; @@ -44,11 +57,13 @@ public SamplingResult shouldSample( parentSpanSampleRate = ((ReadableSpan) parentSpan).getAttribute(AiSemanticAttributes.SAMPLE_RATE); } - + logger.info("caliing internalShouldSample from shouldSample with traceId {}, name {}, spanid {}", + traceId, name, spanId); return internalShouldSample(parentSpanContext, parentSpanSampleRate, traceId); } public SamplingResult shouldSampleLog(SpanContext spanContext, @Nullable Double spanSampleRate) { + logger.info("Calling internalShouldSample from shouldSampleLog with traceId {} and spanId {}", spanContext.getTraceId(), spanContext.getSpanId()); return internalShouldSample(spanContext, spanSampleRate, spanContext.getTraceId()); } @@ -58,10 +73,12 @@ private SamplingResult internalShouldSample( SamplingResult samplingResult = useLocalParentDecisionIfPossible(parentSpanContext, parentSpanSampleRate); if (samplingResult != null) { + logger.info("sampling result: {}", samplingResult.getDecision().toString()); return samplingResult; } - - return SamplerUtil.shouldSample(traceId, percentage); + samplingResult = SamplerUtil.shouldSample(traceId, percentage, quickPulse); + logger.info("sampling result: {}", samplingResult.getDecision().toString()); + return samplingResult;//SamplerUtil.shouldSample(traceId, percentage); } @Nullable @@ -78,6 +95,9 @@ private SamplingResult useLocalParentDecisionIfPossible( if (!parentSpanContext.isSampled()) { if (percentage < 100) { + if (quickPulse != null && quickPulse.isEnabled()) { + return SamplingResult.recordOnly(); + } // only 100% sampling override will override an unsampled parent!! return SamplingResult.drop(); } else { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java index 7d3feadf1e4..134cd4e9d3a 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java @@ -3,8 +3,10 @@ package com.microsoft.applicationinsights.agent.internal.sampling; +import com.azure.core.util.logging.ClientLogger; import com.azure.monitor.opentelemetry.autoconfigure.implementation.AiSemanticAttributes; import com.azure.monitor.opentelemetry.autoconfigure.implementation.RequestChecker; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.QuickPulse; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; @@ -27,24 +29,30 @@ public class AiSampler implements Sampler { private final SamplingPercentage requestSamplingPercentage; private final SamplingPercentage parentlessDependencySamplingPercentage; private final boolean ingestionSamplingEnabled; + private static final ClientLogger logger = new ClientLogger(AiSampler.class); + private final QuickPulse quickPulse; public static AiSampler create( SamplingPercentage requestSamplingPercentage, SamplingPercentage parentlessDependencySamplingPercentage, - boolean ingestionSamplingEnabled) { + boolean ingestionSamplingEnabled, + QuickPulse quickPulse) { return new AiSampler( requestSamplingPercentage, parentlessDependencySamplingPercentage, - ingestionSamplingEnabled); + ingestionSamplingEnabled, + quickPulse); } private AiSampler( SamplingPercentage requestSamplingPercentage, SamplingPercentage parentlessDependencySamplingPercentage, - boolean ingestionSamplingEnabled) { + boolean ingestionSamplingEnabled, + QuickPulse quickPulse) { this.requestSamplingPercentage = requestSamplingPercentage; this.parentlessDependencySamplingPercentage = parentlessDependencySamplingPercentage; this.ingestionSamplingEnabled = ingestionSamplingEnabled; + this.quickPulse = quickPulse; } @Override @@ -55,7 +63,12 @@ public SamplingResult shouldSample( SpanKind spanKind, Attributes attributes, List parentLinks) { - + String spanId = ""; + if (!parentLinks.isEmpty()) { + spanId = parentLinks.get(0).getSpanContext().getSpanId(); + } + logger.info("calling shouldsample from AISampler with traceId {}, name {}, spanId {}", + traceId, name, spanId); Span parentSpan = Span.fromContext(parentContext); SpanContext parentSpanContext = parentSpan.getSpanContext(); Double parentSpanSampleRate = null; @@ -67,6 +80,7 @@ public SamplingResult shouldSample( SamplingResult samplingResult = useLocalParentDecisionIfPossible(parentSpanContext, parentSpanSampleRate); if (samplingResult != null) { + logger.info("sampling result: {}", samplingResult.getDecision().toString()); return samplingResult; } @@ -83,14 +97,17 @@ public SamplingResult shouldSample( } if (sp == 100 && ingestionSamplingEnabled) { + logger.info("sampling result: record and sample"); return SamplingResult.recordAndSample(); } - return SamplerUtil.shouldSample(traceId, sp); + samplingResult = SamplerUtil.shouldSample(traceId, sp, quickPulse); + logger.info("sampling result: {}", samplingResult.getDecision().toString()); + return samplingResult; //SamplerUtil.shouldSample(traceId, sp); } @Nullable - private static SamplingResult useLocalParentDecisionIfPossible( + private SamplingResult useLocalParentDecisionIfPossible( SpanContext parentSpanContext, @Nullable Double parentSpanSampleRate) { // remote parent-based sampling messes up item counts since item count is not propagated in @@ -102,6 +119,9 @@ private static SamplingResult useLocalParentDecisionIfPossible( } if (!parentSpanContext.isSampled()) { + if (quickPulse != null && quickPulse.isEnabled()) { + return SamplingResult.recordOnly(); + } return SamplingResult.drop(); } if (parentSpanSampleRate == null) { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/DelegatingSampler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/DelegatingSampler.java index 9dd2cf202bb..eccbb149cad 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/DelegatingSampler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/DelegatingSampler.java @@ -3,6 +3,7 @@ package com.microsoft.applicationinsights.agent.internal.sampling; +import com.azure.core.util.logging.ClientLogger; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.context.Context; @@ -18,6 +19,8 @@ public class DelegatingSampler implements Sampler { // in Azure Functions consumption pool, we don't know at startup whether to enable or not private volatile Sampler delegate = Sampler.alwaysOff(); + private static final ClientLogger logger = new ClientLogger(DelegatingSampler.class); + public static DelegatingSampler getInstance() { return instance; } @@ -47,6 +50,15 @@ public SamplingResult shouldSample( SpanKind spanKind, Attributes attributes, List parentLinks) { + + String spanId = ""; + if (!parentLinks.isEmpty()) { + spanId = parentLinks.get(0).getSpanContext().getSpanId(); + } + logger.info("calling shouldsample from delegating sampler with traceId {}, name {}, spanid {}", + traceId, name, spanId); + Exception ex = new Exception(); + ex.printStackTrace(); return delegate.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplerUtil.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplerUtil.java index fd810248a09..c4ddaa130e5 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplerUtil.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplerUtil.java @@ -5,6 +5,7 @@ import com.azure.monitor.opentelemetry.autoconfigure.implementation.AiSemanticAttributes; import com.azure.monitor.opentelemetry.autoconfigure.implementation.SamplingScoreGeneratorV2; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.QuickPulse; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.instrumentation.api.internal.cache.Cache; import io.opentelemetry.sdk.trace.samplers.SamplingDecision; @@ -17,13 +18,19 @@ public class SamplerUtil { private static final Cache recordAndSampleWithSampleRateMap = Cache.bounded(100); - static SamplingResult shouldSample(String traceId, double sp) { + static SamplingResult shouldSample(String traceId, double sp, QuickPulse quickPulse) { SamplingResult samplingResult; if (sp == 0) { + if (quickPulse != null && quickPulse.isEnabled()) { + return SamplingResult.recordOnly(); + } return SamplingResult.drop(); } if (sp != 100 && !shouldRecordAndSample(traceId, sp)) { + if (quickPulse != null && quickPulse.isEnabled()) { + return SamplingResult.recordOnly(); + } return SamplingResult.drop(); } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java index 29dc2316026..9c57b7d1993 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java @@ -3,6 +3,7 @@ package com.microsoft.applicationinsights.agent.internal.sampling; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.QuickPulse; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride; import io.opentelemetry.sdk.trace.samplers.Sampler; @@ -12,7 +13,7 @@ public class Samplers { public static Sampler getSampler( - Configuration.Sampling sampling, Configuration.SamplingPreview samplingPreview) { + Configuration.Sampling sampling, Configuration.SamplingPreview samplingPreview, QuickPulse quickPulse) { Sampler sampler; if (sampling.requestsPerSecond != null) { SamplingPercentage requestSamplingPercentage = @@ -22,12 +23,13 @@ public static Sampler getSampler( AiSampler.create( requestSamplingPercentage, parentlessDependencySamplingPercentage, - samplingPreview.ingestionSamplingEnabled); + samplingPreview.ingestionSamplingEnabled, + quickPulse); } else if (sampling.percentage != null) { SamplingPercentage samplingPercentage = SamplingPercentage.fixed(sampling.percentage); sampler = AiSampler.create( - samplingPercentage, samplingPercentage, samplingPreview.ingestionSamplingEnabled); + samplingPercentage, samplingPercentage, samplingPreview.ingestionSamplingEnabled, quickPulse); } else { throw new AssertionError("ConfigurationBuilder should have set the default sampling"); } @@ -44,7 +46,7 @@ public static Sampler getSampler( if (!requestSamplingOverrides.isEmpty() || !dependencySamplingOverrides.isEmpty()) { sampler = new SamplingOverridesSampler( - requestSamplingOverrides, dependencySamplingOverrides, sampler); + requestSamplingOverrides, dependencySamplingOverrides, sampler, quickPulse); } if (!samplingPreview.parentBased) { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java index 79d8e6e43d2..7a13ad3bdd5 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java @@ -4,6 +4,7 @@ package com.microsoft.applicationinsights.agent.internal.sampling; import com.azure.monitor.opentelemetry.autoconfigure.implementation.SpanDataMapper; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.QuickPulse; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.MatchType; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverrideAttribute; @@ -25,10 +26,10 @@ public class SamplingOverrides { private static final Logger logger = LoggerFactory.getLogger(SamplingOverrides.class); private final List matcherGroups; - public SamplingOverrides(List overrides) { + public SamplingOverrides(List overrides, QuickPulse quickPulse) { matcherGroups = new ArrayList<>(); for (SamplingOverride override : overrides) { - matcherGroups.add(new MatcherGroup(override)); + matcherGroups.add(new MatcherGroup(override, quickPulse)); } } @@ -48,7 +49,7 @@ private static class MatcherGroup { private final List predicates; private final AiFixedPercentageSampler sampler; - private MatcherGroup(SamplingOverride override) { + private MatcherGroup(SamplingOverride override, QuickPulse quickPulse) { predicates = new ArrayList<>(); for (SamplingOverrideAttribute attribute : override.attributes) { TempPredicate predicate = toPredicate(attribute); @@ -56,7 +57,7 @@ private MatcherGroup(SamplingOverride override) { predicates.add(predicate); } } - sampler = AiFixedPercentageSampler.create(override.percentage); + sampler = AiFixedPercentageSampler.create(override.percentage, quickPulse); } AiFixedPercentageSampler getSampler() { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesSampler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesSampler.java index f1a99fda03c..bf1ac64c7d4 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesSampler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesSampler.java @@ -3,7 +3,9 @@ package com.microsoft.applicationinsights.agent.internal.sampling; +import com.azure.core.util.logging.ClientLogger; import com.azure.monitor.opentelemetry.autoconfigure.implementation.RequestChecker; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.QuickPulse; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; @@ -20,13 +22,15 @@ class SamplingOverridesSampler implements Sampler { private final SamplingOverrides requestSamplingOverrides; private final SamplingOverrides dependencySamplingOverrides; private final Sampler delegate; + private static final ClientLogger logger = new ClientLogger(SamplingOverridesSampler.class); SamplingOverridesSampler( List requestSamplingOverrides, List dependencySamplingOverrides, - Sampler delegate) { - this.requestSamplingOverrides = new SamplingOverrides(requestSamplingOverrides); - this.dependencySamplingOverrides = new SamplingOverrides(dependencySamplingOverrides); + Sampler delegate, + QuickPulse quickPulse) { + this.requestSamplingOverrides = new SamplingOverrides(requestSamplingOverrides, quickPulse); + this.dependencySamplingOverrides = new SamplingOverrides(dependencySamplingOverrides, quickPulse); this.delegate = delegate; } @@ -38,17 +42,27 @@ public SamplingResult shouldSample( SpanKind spanKind, Attributes attributes, List parentLinks) { + SpanContext parentSpanContext = Span.fromContext(parentContext).getSpanContext(); boolean isRequest = RequestChecker.isRequest(spanKind, parentSpanContext, attributes::get); + logger.info("calling shouldsample from SamplingOverrides Sampler with traceId {}, name {}, parent spanId {}", + traceId, name, parentSpanContext.getSpanId()); + SamplingOverrides samplingOverrides = isRequest ? requestSamplingOverrides : dependencySamplingOverrides; Sampler override = samplingOverrides.getOverride(attributes); if (override != null) { - return override.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + SamplingResult samplingResult = + override.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + logger.info("sampling result {}", samplingResult.getDecision().toString()); + return samplingResult; } - return delegate.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + SamplingResult samplingResult = delegate.shouldSample( + parentContext, traceId, name, spanKind, attributes, parentLinks); + logger.info("sampling result {}", samplingResult.getDecision().toString()); + return samplingResult; } @Override diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java index b31f24107c3..adf1e9ce91c 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java @@ -4,6 +4,7 @@ package com.microsoft.applicationinsights.agent.internal.telemetry; import com.azure.core.http.HttpPipeline; +import com.azure.core.util.logging.ClientLogger; import com.azure.monitor.opentelemetry.autoconfigure.implementation.builders.AbstractTelemetryBuilder; import com.azure.monitor.opentelemetry.autoconfigure.implementation.builders.AvailabilityTelemetryBuilder; import com.azure.monitor.opentelemetry.autoconfigure.implementation.builders.EventTelemetryBuilder; @@ -85,6 +86,8 @@ public class TelemetryClient { @Nullable private volatile BatchItemProcessor metricsBatchItemProcessor; @Nullable private volatile BatchItemProcessor statsbeatBatchItemProcessor; + private static final ClientLogger logger = new ClientLogger(TelemetryClient.class); + public static TelemetryClient.Builder builder() { return new TelemetryClient.Builder(); } @@ -166,6 +169,12 @@ public void trackAsync(TelemetryItem telemetryItem) { } if (quickPulse != null) { + try { + logger.info("Calling quickpulse.add from telemetry client for {} {}", data.getClass(), data.toJsonString()); + } + catch (Exception e) { + logger.warning("Failed to log telemetry item", e); + } quickPulse.add(telemetryItem); } @@ -415,6 +424,10 @@ public void setQuickPulse(@Nullable QuickPulse quickPulse) { this.quickPulse = quickPulse; } + public QuickPulse getQuickPulse() { + return quickPulse; + } + public void setOtelResource(Resource resource) { otelResource = resource; } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesTest.java index 74413533630..eba310bd824 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesTest.java @@ -24,7 +24,7 @@ class SamplingOverridesTest { void shouldSampleByDefault() { // given List overrides = new ArrayList<>(); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.empty(); // when @@ -38,7 +38,7 @@ void shouldSampleByDefault() { void shouldFilterInRequest() { // given List overrides = singletonList(newOverride(25)); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.empty(); // when @@ -54,7 +54,7 @@ void shouldFilterStrictMatch() { // given List overrides = singletonList(newOverride(25, newStrictAttribute("one", "1"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "1"); // when @@ -70,7 +70,7 @@ void shouldNotFilterStrictMatch() { // given List overrides = singletonList(newOverride(25, newStrictAttribute("one", "1"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "2"); // when @@ -85,7 +85,7 @@ void shouldNotFilterMissingStrictMatch() { // given List overrides = singletonList(newOverride(25, newStrictAttribute("one", "1"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("two"), "1"); // when @@ -100,7 +100,7 @@ void shouldFilterRegexpMatch() { // given List overrides = singletonList(newOverride(25, newRegexpAttribute("one", "1.*"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "11"); // when @@ -116,7 +116,7 @@ void shouldNotFilterRegexpMatch() { // given List overrides = singletonList(newOverride(25, newRegexpAttribute("one", "1.*"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "22"); // when @@ -131,7 +131,7 @@ void shouldNotFilterMissingRegexpMatch() { // given List overrides = singletonList(newOverride(25, newRegexpAttribute("one", "1.*"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("two"), "11"); // when @@ -145,7 +145,7 @@ void shouldNotFilterMissingRegexpMatch() { void shouldFilterKeyOnlyMatch() { // given List overrides = singletonList(newOverride(25, newKeyOnlyAttribute("one"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "11"); // when @@ -160,7 +160,7 @@ void shouldFilterKeyOnlyMatch() { void shouldNotFilterKeyOnlyMatch() { // given List overrides = singletonList(newOverride(25, newKeyOnlyAttribute("one"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("two"), "22"); // when @@ -176,7 +176,7 @@ void shouldFilterMultiAttributes() { List overrides = singletonList( newOverride(25, newStrictAttribute("one", "1"), newRegexpAttribute("two", "2.*"))); - SamplingOverrides samplerOverride = new SamplingOverrides(overrides); + SamplingOverrides samplerOverride = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "1", AttributeKey.stringKey("two"), "22"); @@ -194,7 +194,7 @@ void shouldNotFilterMultiAttributes() { List overrides = singletonList( newOverride(25, newStrictAttribute("one", "1"), newRegexpAttribute("two", "2.*"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "2", AttributeKey.stringKey("two"), "22"); @@ -212,7 +212,7 @@ void shouldFilterMultiConfigsBothMatch() { Arrays.asList( newOverride(25, newStrictAttribute("one", "1")), newOverride(0, newRegexpAttribute("two", "2.*"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "1", AttributeKey.stringKey("two"), "22"); @@ -231,7 +231,7 @@ void shouldFilterMultiConfigsOneMatch() { Arrays.asList( newOverride(50, newStrictAttribute("one", "1")), newOverride(25, newRegexpAttribute("two", "2.*"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "2", AttributeKey.stringKey("two"), "22"); @@ -250,7 +250,7 @@ void shouldNotFilterMultiConfigsNoMatch() { Arrays.asList( newOverride(50, newStrictAttribute("one", "1")), newOverride(25, newRegexpAttribute("two", "2.*"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); + SamplingOverrides samplingOverrides = new SamplingOverrides(overrides, null); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "2", AttributeKey.stringKey("two"), "33");