OpenTelemetry Tracing: Visualize JGroups communication (#39659)

Closes #39658

Signed-off-by: Alexander Schwartz <[email protected]>

Author: ahus1

docs/documentation/upgrading/topics/changes/changes-26_4_0.adoc

@@ -63,6 +63,13 @@ In order to maintain backwards compatibility, {project_name}'s upgrade will modi
 
 For more information about client configuration, please see link:{adminguide_link}#_client-saml-configuration[Creating a SAML client] chapter in the {adminguide_name}.
 
+=== Tracing extended for embedded Infinispan caches
+
+When tracing is enabled, now also calls to other nodes of a {project_name} cluster will create spans in the traces.
+
+To disable this kind of tracing, set the option `tracing-infinispan-enabled` to `false`.
+
+
 // ------------------------ Deprecated features ------------------------ //
 == Deprecated features
 

model/infinispan/src/main/java/org/keycloak/infinispan/module/factory/InfinispanTelemetryFactory.java

@@ -0,0 +1,34 @@
+package org.keycloak.infinispan.module.factory;
+
+import io.opentelemetry.api.OpenTelemetry;
+import jakarta.enterprise.inject.Instance;
+import jakarta.enterprise.inject.spi.CDI;
+import org.infinispan.factories.AbstractComponentFactory;
+import org.infinispan.factories.AutoInstantiableFactory;
+import org.infinispan.factories.annotations.DefaultFactoryFor;
+import org.infinispan.factories.scopes.Scope;
+import org.infinispan.factories.scopes.Scopes;
+import org.infinispan.telemetry.InfinispanTelemetry;
+import org.infinispan.telemetry.impl.DisabledInfinispanTelemetry;
+
+@Scope(Scopes.GLOBAL)
+@DefaultFactoryFor(classes = InfinispanTelemetry.class)
+public class InfinispanTelemetryFactory extends AbstractComponentFactory implements AutoInstantiableFactory {
+
+    @Override
+    public Object construct(String componentName) {
+        CDI<Object> current;
+        try {
+            current = CDI.current();
+        } catch (IllegalStateException e) {
+            // No CDI context, assume tracing is not available
+            return new DisabledInfinispanTelemetry();
+        }
+        Instance<OpenTelemetry> selector = current.select(OpenTelemetry.class);
+        if (!selector.isResolvable()) {
+            return new DisabledInfinispanTelemetry();
+        } else {
+            return new OpenTelemetryService(selector.get());
+        }
+    }
+}

model/infinispan/src/main/java/org/keycloak/infinispan/module/factory/OpenTelemetryService.java

@@ -0,0 +1,74 @@
+package org.keycloak.infinispan.module.factory;
+
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanBuilder;
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.propagation.TextMapGetter;
+import org.infinispan.telemetry.InfinispanSpan;
+import org.infinispan.telemetry.InfinispanSpanAttributes;
+import org.infinispan.telemetry.InfinispanSpanContext;
+import org.infinispan.telemetry.InfinispanTelemetry;
+
+public class OpenTelemetryService implements InfinispanTelemetry, TextMapGetter<InfinispanSpanContext> {
+
+    private static final String INFINISPAN_SERVER_TRACING_NAME = "org.infinispan.server.tracing";
+    private static final String INFINISPAN_SERVER_TRACING_VERSION = "1.0.0";
+
+    private final Tracer tracer;
+    private volatile String nodeName = "n/a";
+
+    public OpenTelemetryService(OpenTelemetry openTelemetry) {
+        this.tracer = openTelemetry.getTracer(INFINISPAN_SERVER_TRACING_NAME, INFINISPAN_SERVER_TRACING_VERSION);
+    }
+
+    @Override
+    public <T> InfinispanSpan<T> startTraceRequest(String operationName, InfinispanSpanAttributes attributes) {
+        // The original Infininspan implementation allows for filtering via the trace attributes. We don't support this here and instead trace everything.
+
+        var builder = tracer.spanBuilder(operationName)
+                .setSpanKind(SpanKind.SERVER);
+        // the parent context is inherited automatically,
+        // because the parent span is created in the same process
+
+        return createOpenTelemetrySpan(builder, attributes);
+    }
+
+    @Override
+    public <T> InfinispanSpan<T> startTraceRequest(String operationName, InfinispanSpanAttributes attributes, InfinispanSpanContext context) {
+        // The original Infinispan implementation allows for filtering via the trace attributes. We don't support this here and instead trace everything
+
+        var builder = tracer.spanBuilder(operationName)
+                .setSpanKind(SpanKind.SERVER)
+                .setParent(Context.current().with(Span.current()));
+
+        return createOpenTelemetrySpan(builder, attributes);
+    }
+
+    @Override
+    public void setNodeName(String nodeName) {
+        if (nodeName != null) {
+            this.nodeName = nodeName;
+        }
+    }
+
+    private <T> InfinispanSpan<T> createOpenTelemetrySpan(SpanBuilder builder, InfinispanSpanAttributes attributes) {
+        attributes.cacheName().ifPresent(cacheName -> builder.setAttribute("cache", cacheName));
+        builder.setAttribute("category", attributes.category().toString());
+        builder.setAttribute("server.address", nodeName);
+        return new OpenTelemetrySpan<>(builder.startSpan());
+    }
+
+    @Override
+    public Iterable<String> keys(InfinispanSpanContext ctx) {
+        return ctx.keys();
+    }
+
+    @Override
+    public String get(InfinispanSpanContext ctx, String key) {
+        assert ctx != null;
+        return ctx.getKey(key);
+    }
+}

model/infinispan/src/main/java/org/keycloak/infinispan/module/factory/OpenTelemetrySpan.java

@@ -0,0 +1,42 @@
+package org.keycloak.infinispan.module.factory;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.StatusCode;
+import io.opentelemetry.context.Scope;
+import org.infinispan.telemetry.InfinispanSpan;
+import org.infinispan.telemetry.SafeAutoClosable;
+
+import java.util.Objects;
+
+public class OpenTelemetrySpan<T> implements InfinispanSpan<T> {
+
+    private final Span span;
+    private final Scope scope;
+
+    public OpenTelemetrySpan(Span span) {
+        this.span = Objects.requireNonNull(span);
+        // TODO: This is actually wrong if you are doing asynchronous calls, but it allows the JGroups calls to be nested
+        // This should be fixed in ISPN 16+ so that it is no longer needed
+        // https://github.com/infinispan/infinispan/issues/15287
+        this.scope = span.makeCurrent();
+    }
+
+    @Override
+    public SafeAutoClosable makeCurrent() {
+        //noinspection resource
+        Scope scope = span.makeCurrent();
+        return scope::close;
+    }
+
+    @Override
+    public void complete() {
+        scope.close();
+        span.end();
+    }
+
+    @Override
+    public void recordException(Throwable throwable) {
+        span.setStatus(StatusCode.ERROR, "Error during the cache request processing");
+        span.recordException(throwable);
+    }
+}

model/infinispan/src/main/java/org/keycloak/jgroups/header/TracerHeader.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2025 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.jgroups.header;
+
+import org.jgroups.Header;
+import org.jgroups.util.Util;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+
+/**
+ * Header which carries an OpenTelemetry {@link io.opentelemetry.api.trace.Span} between requests and responses
+ *
+ * @author Bela Ban
+ * @since 1.0.0
+ */
+public class TracerHeader extends Header {
+    public static final short ID = 1050;
+    protected final Map<String, String> ctx = new HashMap<>();
+
+    public TracerHeader() {
+    }
+
+    public short getMagicId() {
+        return ID;
+    }
+
+    public Supplier<? extends Header> create() {
+        return TracerHeader::new;
+    }
+
+    public void put(String key, String value) {
+        ctx.put(key, value);
+    }
+
+    public String get(String key) {
+        return ctx.get(key);
+    }
+
+    public Set<String> keys() {
+        return ctx.keySet();
+    }
+
+    public int serializedSize() {
+        int size = Integer.BYTES;
+        int num_attrs = ctx.size();
+        if (num_attrs > 0) {
+            for (Map.Entry<String, String> entry : ctx.entrySet()) {
+                String key = entry.getKey();
+                String val = entry.getValue();
+                size += Util.size(key) + Util.size(val);
+            }
+        }
+        return size;
+    }
+
+    public void writeTo(DataOutput out) throws IOException {
+        out.writeInt(ctx.size());
+        if (!ctx.isEmpty()) {
+            for (Map.Entry<String, String> e : ctx.entrySet()) {
+                Util.writeString(e.getKey(), out);
+                Util.writeString(e.getValue(), out);
+            }
+        }
+    }
+
+    public void readFrom(DataInput in) throws IOException {
+        int size = in.readInt();
+        if (size > 0) {
+            for (int i = 0; i < size; i++)
+                ctx.put(Util.readString(in), Util.readString(in));
+        }
+    }
+
+    public String toString() {
+        return ctx.toString();
+    }
+}