Tracing for tests (with OTEL Agent)

solr/core/src/java/org/apache/solr/core/SolrCore.java

@@ -176,6 +176,7 @@
 import org.apache.solr.util.plugin.NamedListInitializedPlugin;
 import org.apache.solr.util.plugin.PluginInfoInitialized;
 import org.apache.solr.util.plugin.SolrCoreAware;
+import org.apache.solr.util.tracing.TraceUtils;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.data.Stat;
 import org.eclipse.jetty.io.RuntimeIOException;
@@ -2964,6 +2965,7 @@ public String[] getParams(String param) { // assume param is in lpSet
   /** Put status, QTime, and possibly request handler and params, in the response header */
   public static void postDecorateResponse(
       SolrRequestHandler handler, SolrQueryRequest req, SolrQueryResponse rsp) {
+    TraceUtils.ifNotNoop(req.getSpan(), span -> TraceUtils.postDecorate(span, req, rsp));
     // TODO should check that responseHeader has not been replaced by handler
     NamedList<Object> responseHeader = rsp.getResponseHeader();
     if (responseHeader == null) return;

solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java

@@ -28,14 +28,18 @@
 import io.opentelemetry.context.propagation.TextMapSetter;
 import jakarta.servlet.http.HttpServletRequest;
 import java.util.List;
+import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
+import org.apache.solr.common.util.NamedList;
 import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
 import org.eclipse.jetty.client.Request;
 
 /** Utilities for distributed tracing. */
 public class TraceUtils {
 
+  private static final Set<String> SKIP_REQLOG = Set.of("webapp", "params", "path");
   private static final String REQ_ATTR_TRACING_SPAN = Span.class.getName();
   private static final String REQ_ATTR_TRACING_TRACER = Tracer.class.getName();
 
@@ -185,4 +189,42 @@ public static Span startCollectionApiCommandSpan(
         tracer.spanBuilder(name).setSpanKind(kind).setAttribute(TAG_DB, collection);
     return spanBuilder.startSpan();
   }
+
+  public static void postDecorate(Span span, SolrQueryRequest req, SolrQueryResponse rsp) {
+    // assume we are "recording"
+
+    // TODO perf: add overloaded setAttribute(String, Object)
+
+    // Add a very human readable params; one param per line with no URL encoding.
+    //   This is somewhat redundant with http.params but that one's sometimes too hard to use.
+    StringBuilder sb = new StringBuilder(256);
+    req.getParams()
+        .forEach(
+            (entry) -> {
+              // nocommit remember CommonParams.LOG_PARAMS_LIST
+              for (String v : entry.getValue()) {
+                sb.append(entry.getKey()).append('=').append(v).append('\n');
+              }
+            });
+    span.setAttribute("solr.request.params", sb.toString());
+
+    // Add an attribute for each rsp.getToLog
+    rsp.getToLog()
+        .forEach(
+            (k, v) -> {
+              if (!SKIP_REQLOG.contains(k)) {
+                span.setAttribute("solr.reqlog." + k, String.valueOf(v));
+              }
+            });
+
+    // Add an attribute for each Solr response header
+    // TODO filter out how?
+    NamedList<Object> responseHeader = rsp.getResponseHeader();
+    if (responseHeader != null) {
+      responseHeader.forEach(
+          (k, v) -> {
+            span.setAttribute("solr.response." + k, String.valueOf(v));
+          });
+    }
+  }
 }

solr/modules/opentelemetry/README.md

@@ -23,4 +23,61 @@ Introduction
 This module brings support for the new [OTEL](https://opentelemetry.io) (OpenTelemetry) standard,
 and exposes a tracer configurator that can be enabled in the
 `<tracerConfig>` tag of `solr.xml`. Please see Solr Reference Guide chapter "Distributed Tracing"
-for details.
+for details.
+
+Enabling Tracing in Tests
+--------------------------
+
+Here's a tip to enable distributed tracing in a Solr test:
+
+
+### Step 1: Download the OpenTelemetry Java Agent
+
+Run the Gradle task to download the agent JAR:
+
+```bash
+./gradlew :solr:modules:opentelemetry:downloadOtelAgent
+```
+
+This downloads the agent to: `solr/modules/opentelemetry/build/agent/opentelemetry-javaagent.jar`
+
+The file is gitignored (build directory) so you'll need to run this task once per checkout or after clean builds.
+
+### Step 2: Set Up a Tracing Backend (e.g., Jaeger)
+
+Start Jaeger using Docker:
+
+```bash
+docker run --rm --name jaeger \
+  -p 16686:16686 -p 4317:4317 \
+  cr.jaegertracing.io/jaegertracing/jaeger:2.10.0
+```
+
+Access Jaeger UI at: http://localhost:16686
+
+### Step 3: Configure IntelliJ Run Configuration
+
+1. **Open Run/Debug Configurations**
+    - Click on the configuration dropdown in the toolbar
+    - Select "Edit Configurations..."
+
+2. **Select or Create a Test Configuration**
+    - Select an existing JUnit test configuration, or
+    - Click "+" to create a new JUnit configuration
+
+3. **Add VM Options**
+    - In the "VM options" field, add:
+   ```
+   -javaagent:$PROJECT_DIR$/solr/modules/opentelemetry/build/agent/opentelemetry-javaagent.jar
+   -Dotel.javaagent.configuration-file=$PROJECT_DIR$/solr/modules/opentelemetry/otel-agent-test-config.properties
+   ```
+
+4. **Apply and Save**
+    - Click "Apply" then "OK"
+
+### Step 4: Run Tests with Tracing
+
+1. Run your test configuration as normal
+2. Open Jaeger UI at http://localhost:16686
+3. Select service "solr-test" from the dropdown
+4. Click "Find Traces" to see your test execution traces

solr/modules/opentelemetry/build.gradle

@@ -50,3 +50,27 @@ dependencies {
   testImplementation libs.opentelemetry.sdktrace
   testImplementation libs.opentelemetry.sdktesting
 }
+
+// Configuration for downloading the OpenTelemetry Java Agent
+configurations {
+  otelAgent
+}
+
+dependencies {
+  otelAgent "io.opentelemetry.javaagent:opentelemetry-javaagent:[2.0,3.0)"
+}
+
+task downloadOtelAgent(type: Copy) {
+  description = 'Downloads the OpenTelemetry Java Agent JAR for use with IntelliJ run configurations'
+  group = 'build setup'
+
+  from configurations.otelAgent
+  into layout.buildDirectory.dir('agent')
+  rename { filename ->
+    'opentelemetry-javaagent.jar'
+  }
+
+  doLast {
+    logger.lifecycle("OpenTelemetry Java Agent downloaded to: ${layout.buildDirectory.get()}/agent/opentelemetry-javaagent.jar")
+  }
+}

solr/modules/opentelemetry/otel-agent-test-config.properties

@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# OpenTelemetry Java Agent Configuration for Solr Tests
+# This file provides default settings for using the OTEL Java Agent with IntelliJ run configurations.
+# Reference: https://opentelemetry.io/docs/languages/java/configuration/
+
+# Service name for trace identification
+otel.service.name=solr-test
+
+# Enable traces, disable metrics and logs
+otel.traces.exporter=otlp
+otel.metrics.exporter=none
+otel.logs.exporter=none
+
+# OTLP exporter configuration (gRPC)
+otel.exporter.otlp.protocol=grpc
+otel.exporter.otlp.endpoint=http://localhost:4317
+
+# Sampling strategy - use "always_on" for comprehensive test coverage
+# For production, consider "parentbased_always_on" or ratio-based sampling
+otel.traces.sampler=always_on
+
+# Propagators for distributed tracing
+otel.propagators=tracecontext
+
+# SDK configuration
+otel.sdk.disabled=false
+
+# For debugging the agent itself:
+otel.javaagent.debug=false
+
+# Resource attributes stuff
+otel.java.disabled.resource.providers=io.opentelemetry.instrumentation.resources.ProcessResourceProvider
+
+# Instrumentation configuration
+#      https://opentelemetry.io/docs/zero-code/java/agent/disable/
+
+# Setting to default to false allows opt-in tracing of instrumentation we care about.
+otel.instrumentation.common.default-enabled=false
+
+# By default, the agent auto-instruments many libraries
+# You can disable specific instrumentations if needed:
+# otel.instrumentation.[name].enabled=false
+otel.instrumentation.opentelemetry-api.enabled=true
+otel.instrumentation.opentelemetry-instrumentation-annotations.enabled=true
+
+# my.package.MyClass1[method1,method2];my.package.MyClass2[method3]
+#otel.instrumentation.methods.enabled=true
+#otel.instrumentation.methods.include