From bd1748846a7adc6ca0c01eedec9384bd2e67406d Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 3 Oct 2025 01:06:28 -0400 Subject: [PATCH 1/2] Tracing agent instructions --- solr/modules/opentelemetry/README.md | 59 ++++++++++++++++- solr/modules/opentelemetry/build.gradle | 24 +++++++ .../otel-agent-test-config.properties | 64 +++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 solr/modules/opentelemetry/otel-agent-test-config.properties diff --git a/solr/modules/opentelemetry/README.md b/solr/modules/opentelemetry/README.md index 9e7a1a65d84..339afb08607 100644 --- a/solr/modules/opentelemetry/README.md +++ b/solr/modules/opentelemetry/README.md @@ -23,4 +23,61 @@ Introduction This module brings support for the new [OTEL](https://opentelemetry.io) (OpenTelemetry) standard, and exposes a tracer configurator that can be enabled in the `` tag of `solr.xml`. Please see Solr Reference Guide chapter "Distributed Tracing" -for details. \ No newline at end of file +for details. + +Enabling Tracing in Tests +-------------------------- + +Here's a tip to enable distributed tracing in a Solr test: + + +### Step 1: Download the OpenTelemetry Java Agent + +Run the Gradle task to download the agent JAR: + +```bash +./gradlew :solr:modules:opentelemetry:downloadOtelAgent +``` + +This downloads the agent to: `solr/modules/opentelemetry/build/agent/opentelemetry-javaagent.jar` + +The file is gitignored (build directory) so you'll need to run this task once per checkout or after clean builds. + +### Step 2: Set Up a Tracing Backend (e.g., Jaeger) + +Start Jaeger using Docker: + +```bash +docker run --rm --name jaeger \ + -p 16686:16686 -p 4317:4317 \ + cr.jaegertracing.io/jaegertracing/jaeger:2.10.0 +``` + +Access Jaeger UI at: http://localhost:16686 + +### Step 3: Configure IntelliJ Run Configuration + +1. **Open Run/Debug Configurations** + - Click on the configuration dropdown in the toolbar + - Select "Edit Configurations..." + +2. **Select or Create a Test Configuration** + - Select an existing JUnit test configuration, or + - Click "+" to create a new JUnit configuration + +3. **Add VM Options** + - In the "VM options" field, add: + ``` + -javaagent:$PROJECT_DIR$/solr/modules/opentelemetry/build/agent/opentelemetry-javaagent.jar + -Dotel.javaagent.configuration-file=$PROJECT_DIR$/solr/modules/opentelemetry/otel-agent-test-config.properties + ``` + +4. **Apply and Save** + - Click "Apply" then "OK" + +### Step 4: Run Tests with Tracing + +1. Run your test configuration as normal +2. Open Jaeger UI at http://localhost:16686 +3. Select service "solr-test" from the dropdown +4. Click "Find Traces" to see your test execution traces diff --git a/solr/modules/opentelemetry/build.gradle b/solr/modules/opentelemetry/build.gradle index 7ab50a32421..c509ba7ecb1 100644 --- a/solr/modules/opentelemetry/build.gradle +++ b/solr/modules/opentelemetry/build.gradle @@ -50,3 +50,27 @@ dependencies { testImplementation libs.opentelemetry.sdktrace testImplementation libs.opentelemetry.sdktesting } + +// Configuration for downloading the OpenTelemetry Java Agent +configurations { + otelAgent +} + +dependencies { + otelAgent "io.opentelemetry.javaagent:opentelemetry-javaagent:[2.0,3.0)" +} + +task downloadOtelAgent(type: Copy) { + description = 'Downloads the OpenTelemetry Java Agent JAR for use with IntelliJ run configurations' + group = 'build setup' + + from configurations.otelAgent + into layout.buildDirectory.dir('agent') + rename { filename -> + 'opentelemetry-javaagent.jar' + } + + doLast { + logger.lifecycle("OpenTelemetry Java Agent downloaded to: ${layout.buildDirectory.get()}/agent/opentelemetry-javaagent.jar") + } +} diff --git a/solr/modules/opentelemetry/otel-agent-test-config.properties b/solr/modules/opentelemetry/otel-agent-test-config.properties new file mode 100644 index 00000000000..aaf4098ba8b --- /dev/null +++ b/solr/modules/opentelemetry/otel-agent-test-config.properties @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# OpenTelemetry Java Agent Configuration for Solr Tests +# This file provides default settings for using the OTEL Java Agent with IntelliJ run configurations. +# Reference: https://opentelemetry.io/docs/languages/java/configuration/ + +# Service name for trace identification +otel.service.name=solr-test + +# Enable traces, disable metrics and logs +otel.traces.exporter=otlp +otel.metrics.exporter=none +otel.logs.exporter=none + +# OTLP exporter configuration (gRPC) +otel.exporter.otlp.protocol=grpc +otel.exporter.otlp.endpoint=http://localhost:4317 + +# Sampling strategy - use "always_on" for comprehensive test coverage +# For production, consider "parentbased_always_on" or ratio-based sampling +otel.traces.sampler=always_on + +# Propagators for distributed tracing +otel.propagators=tracecontext + +# SDK configuration +otel.sdk.disabled=false + +# For debugging the agent itself: +otel.javaagent.debug=false + +# Resource attributes stuff +otel.java.disabled.resource.providers=io.opentelemetry.instrumentation.resources.ProcessResourceProvider + +# Instrumentation configuration +# https://opentelemetry.io/docs/zero-code/java/agent/disable/ + +# Setting to default to false allows opt-in tracing of instrumentation we care about. +otel.instrumentation.common.default-enabled=false + +# By default, the agent auto-instruments many libraries +# You can disable specific instrumentations if needed: +# otel.instrumentation.[name].enabled=false +otel.instrumentation.opentelemetry-api.enabled=true +otel.instrumentation.opentelemetry-instrumentation-annotations.enabled=true + +# my.package.MyClass1[method1,method2];my.package.MyClass2[method3] +#otel.instrumentation.methods.enabled=true +#otel.instrumentation.methods.include From e63b204478ed426eca654ed4d7b6df64b3f7fc0a Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 3 Oct 2025 01:08:34 -0400 Subject: [PATCH 2/2] Span detail: params, reqlog, response headers --- .../java/org/apache/solr/core/SolrCore.java | 2 + .../apache/solr/util/tracing/TraceUtils.java | 42 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index dbfe8f7ee0a..98e5d9d34f9 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -176,6 +176,7 @@ import org.apache.solr.util.plugin.NamedListInitializedPlugin; import org.apache.solr.util.plugin.PluginInfoInitialized; import org.apache.solr.util.plugin.SolrCoreAware; +import org.apache.solr.util.tracing.TraceUtils; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; import org.eclipse.jetty.io.RuntimeIOException; @@ -2964,6 +2965,7 @@ public String[] getParams(String param) { // assume param is in lpSet /** Put status, QTime, and possibly request handler and params, in the response header */ public static void postDecorateResponse( SolrRequestHandler handler, SolrQueryRequest req, SolrQueryResponse rsp) { + TraceUtils.ifNotNoop(req.getSpan(), span -> TraceUtils.postDecorate(span, req, rsp)); // TODO should check that responseHeader has not been replaced by handler NamedList responseHeader = rsp.getResponseHeader(); if (responseHeader == null) return; diff --git a/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java b/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java index 11adf3f0eca..7332011ba1a 100644 --- a/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java +++ b/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java @@ -28,14 +28,18 @@ import io.opentelemetry.context.propagation.TextMapSetter; import jakarta.servlet.http.HttpServletRequest; import java.util.List; +import java.util.Set; import java.util.function.Consumer; import java.util.function.Predicate; +import org.apache.solr.common.util.NamedList; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; import org.eclipse.jetty.client.Request; /** Utilities for distributed tracing. */ public class TraceUtils { + private static final Set SKIP_REQLOG = Set.of("webapp", "params", "path"); private static final String REQ_ATTR_TRACING_SPAN = Span.class.getName(); private static final String REQ_ATTR_TRACING_TRACER = Tracer.class.getName(); @@ -185,4 +189,42 @@ public static Span startCollectionApiCommandSpan( tracer.spanBuilder(name).setSpanKind(kind).setAttribute(TAG_DB, collection); return spanBuilder.startSpan(); } + + public static void postDecorate(Span span, SolrQueryRequest req, SolrQueryResponse rsp) { + // assume we are "recording" + + // TODO perf: add overloaded setAttribute(String, Object) + + // Add a very human readable params; one param per line with no URL encoding. + // This is somewhat redundant with http.params but that one's sometimes too hard to use. + StringBuilder sb = new StringBuilder(256); + req.getParams() + .forEach( + (entry) -> { + // nocommit remember CommonParams.LOG_PARAMS_LIST + for (String v : entry.getValue()) { + sb.append(entry.getKey()).append('=').append(v).append('\n'); + } + }); + span.setAttribute("solr.request.params", sb.toString()); + + // Add an attribute for each rsp.getToLog + rsp.getToLog() + .forEach( + (k, v) -> { + if (!SKIP_REQLOG.contains(k)) { + span.setAttribute("solr.reqlog." + k, String.valueOf(v)); + } + }); + + // Add an attribute for each Solr response header + // TODO filter out how? + NamedList responseHeader = rsp.getResponseHeader(); + if (responseHeader != null) { + responseHeader.forEach( + (k, v) -> { + span.setAttribute("solr.response." + k, String.valueOf(v)); + }); + } + } }