[improve][monitor] Upgrade OpenTelemetry to 1.56.0, Otel instrumentation to 2.21.0 and Otel semconv to 1.37.0 (#24994)

Author: lhotari

distribution/server/src/assemble/LICENSE.bin.txt

@@ -339,12 +339,12 @@ The Apache Software License, Version 2.0
     - io.prometheus-simpleclient_tracer_otel-0.16.0.jar
     - io.prometheus-simpleclient_tracer_otel_agent-0.16.0.jar
  * Prometheus exporter
-    - io.prometheus-prometheus-metrics-config-1.3.4.jar
-    - io.prometheus-prometheus-metrics-exporter-common-1.3.4.jar
-    - io.prometheus-prometheus-metrics-exporter-httpserver-1.3.4.jar
-    - io.prometheus-prometheus-metrics-exposition-formats-1.3.4.jar
-    - io.prometheus-prometheus-metrics-model-1.3.4.jar
-    - io.prometheus-prometheus-metrics-exposition-textformats-1.3.4.jar
+    - io.prometheus-prometheus-metrics-config-1.3.10.jar
+    - io.prometheus-prometheus-metrics-exporter-common-1.3.10.jar
+    - io.prometheus-prometheus-metrics-exporter-httpserver-1.3.10.jar
+    - io.prometheus-prometheus-metrics-exposition-formats-no-protobuf-1.3.10.jar
+    - io.prometheus-prometheus-metrics-model-1.3.10.jar
+    - io.prometheus-prometheus-metrics-exposition-textformats-1.3.10.jar
  * Jakarta Bean Validation API
     - jakarta.validation-jakarta.validation-api-2.0.2.jar
     - javax.validation-validation-api-1.1.0.Final.jar
@@ -418,17 +418,16 @@ The Apache Software License, Version 2.0
  * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.38.0.jar
  * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar
  * OkHttp3
-     - com.squareup.okhttp3-logging-interceptor-4.9.3.jar
-     - com.squareup.okhttp3-okhttp-4.9.3.jar
+     - com.squareup.okhttp3-logging-interceptor-5.3.1.jar
+     - com.squareup.okhttp3-okhttp-5.3.1.jar
+     - com.squareup.okhttp3-okhttp-jvm-5.3.1.jar
  * Okio
-     - com.squareup.okio-okio-3.4.0.jar
-     - com.squareup.okio-okio-jvm-3.4.0.jar
+     - com.squareup.okio-okio-3.16.3.jar
+     - com.squareup.okio-okio-jvm-3.16.3.jar
  * Javassist -- org.javassist-javassist-3.25.0-GA.jar
  * Kotlin Standard Lib
      - org.jetbrains.kotlin-kotlin-stdlib-1.8.20.jar
      - org.jetbrains.kotlin-kotlin-stdlib-common-1.8.20.jar
-     - org.jetbrains.kotlin-kotlin-stdlib-jdk7-1.8.20.jar
-     - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.8.20.jar
      - org.jetbrains-annotations-13.0.jar
  * gRPC
     - io.grpc-grpc-all-1.75.0.jar
@@ -519,27 +518,28 @@ The Apache Software License, Version 2.0
   * RoaringBitmap
     - org.roaringbitmap-RoaringBitmap-1.2.0.jar
   * OpenTelemetry
-    - io.opentelemetry-opentelemetry-api-1.45.0.jar
-    - io.opentelemetry-opentelemetry-api-incubator-1.45.0-alpha.jar
-    - io.opentelemetry-opentelemetry-context-1.45.0.jar
-    - io.opentelemetry-opentelemetry-exporter-common-1.45.0.jar
-    - io.opentelemetry-opentelemetry-exporter-otlp-1.45.0.jar
-    - io.opentelemetry-opentelemetry-exporter-otlp-common-1.45.0.jar
-    - io.opentelemetry-opentelemetry-exporter-prometheus-1.45.0-alpha.jar
-    - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-common-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-logs-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-metrics-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-trace-1.45.0.jar
-    - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.33.6.jar
-    - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.33.6-alpha.jar
-    - io.opentelemetry.instrumentation-opentelemetry-resources-1.33.6-alpha.jar
-    - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-1.33.6-alpha.jar
-    - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-1.33.6-alpha.jar
-    - io.opentelemetry.semconv-opentelemetry-semconv-1.29.0-alpha.jar
+    - io.opentelemetry-opentelemetry-api-1.56.0.jar
+    - io.opentelemetry-opentelemetry-api-incubator-1.56.0-alpha.jar
+    - io.opentelemetry-opentelemetry-common-1.56.0.jar
+    - io.opentelemetry-opentelemetry-context-1.56.0.jar
+    - io.opentelemetry-opentelemetry-exporter-common-1.56.0.jar
+    - io.opentelemetry-opentelemetry-exporter-otlp-1.56.0.jar
+    - io.opentelemetry-opentelemetry-exporter-otlp-common-1.56.0.jar
+    - io.opentelemetry-opentelemetry-exporter-prometheus-1.56.0-alpha.jar
+    - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-common-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-logs-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-metrics-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-trace-1.56.0.jar
+    - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-2.21.0.jar
+    - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-incubator-2.21.0-alpha.jar
+    - io.opentelemetry.instrumentation-opentelemetry-resources-2.21.0-alpha.jar
+    - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-2.21.0-alpha.jar
+    - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-2.21.0-alpha.jar
+    - io.opentelemetry.semconv-opentelemetry-semconv-1.37.0.jar
     - com.google.cloud.opentelemetry-detector-resources-support-0.33.0.jar
     - io.opentelemetry.contrib-opentelemetry-gcp-resources-1.48.0-alpha.jar
   * Spotify completable-futures

distribution/shell/src/assemble/LICENSE.bin.txt

@@ -389,9 +389,10 @@ The Apache Software License, Version 2.0
     - log4j-slf4j2-impl-2.25.2.jar
     - log4j-web-2.25.2.jar
  * OpenTelemetry
-    - opentelemetry-api-1.45.0.jar
-    - opentelemetry-api-incubator-1.45.0-alpha.jar
-    - opentelemetry-context-1.45.0.jar
+    - opentelemetry-api-1.56.0.jar
+    - opentelemetry-api-incubator-1.56.0-alpha.jar
+    - opentelemetry-common-1.56.0.jar
+    - opentelemetry-context-1.56.0.jar
 
  * BookKeeper
     - bookkeeper-common-allocator-4.17.2.jar

pom.xml

@@ -285,9 +285,9 @@ flexible messaging model and an intuitive client API.</description>
     <jna.version>5.12.1</jna.version>
     <kubernetesclient.version>18.0.0</kubernetesclient.version>
     <jose4j.version>0.9.4</jose4j.version>
-    <okhttp3.version>4.9.3</okhttp3.version>
+    <okhttp3.version>5.3.1</okhttp3.version>
     <!-- use okio version that matches the okhttp3 version -->
-    <okio.version>3.4.0</okio.version>
+    <okio.version>3.16.3</okio.version>
     <!-- override kotlin-stdlib used by okio in order to address CVE-2020-29582 -->
     <kotlin-stdlib.version>1.8.20</kotlin-stdlib.version>
     <nsq-client.version>1.0</nsq-client.version>
@@ -306,11 +306,11 @@ flexible messaging model and an intuitive client API.</description>
     the core logic is switched to java implementation of zstd in org.apache.commons:commons-compress -->
     <zstd-jni.version>1.5.7-3</zstd-jni.version>
     <netty-reactive-streams.version>2.0.6</netty-reactive-streams.version>
-    <opentelemetry.version>1.45.0</opentelemetry.version>
+    <opentelemetry.version>1.56.0</opentelemetry.version>
     <opentelemetry.alpha.version>${opentelemetry.version}-alpha</opentelemetry.alpha.version>
-    <opentelemetry.instrumentation.version>1.33.6</opentelemetry.instrumentation.version>
+    <opentelemetry.instrumentation.version>2.21.0</opentelemetry.instrumentation.version>
     <opentelemetry.instrumentation.alpha.version>${opentelemetry.instrumentation.version}-alpha</opentelemetry.instrumentation.alpha.version>
-    <opentelemetry.semconv.version>1.29.0-alpha</opentelemetry.semconv.version>
+    <opentelemetry.semconv.version>1.37.0</opentelemetry.semconv.version>
     <picocli.version>4.7.5</picocli.version>
     <re2j.version>1.8</re2j.version>
     <completable-futures.version>0.3.6</completable-futures.version>
@@ -1631,23 +1631,17 @@ flexible messaging model and an intuitive client API.</description>
 
       <dependency>
         <groupId>com.squareup.okhttp3</groupId>
-        <artifactId>okhttp</artifactId>
-        <version>${okhttp3.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>com.squareup.okhttp3</groupId>
-        <artifactId>okhttp-urlconnection</artifactId>
-        <version>${okhttp3.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>com.squareup.okhttp3</groupId>
-        <artifactId>logging-interceptor</artifactId>
+        <artifactId>okhttp-bom</artifactId>
         <version>${okhttp3.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
       </dependency>
       <dependency>
         <groupId>com.squareup.okio</groupId>
-        <artifactId>okio</artifactId>
+        <artifactId>okio-bom</artifactId>
         <version>${okio.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
       </dependency>
 
       <dependency>

pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java

@@ -28,7 +28,7 @@
 import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder;
 import io.opentelemetry.sdk.common.export.MemoryMode;
 import io.opentelemetry.sdk.resources.Resource;
-import io.opentelemetry.semconv.ResourceAttributes;
+import io.opentelemetry.semconv.ServiceAttributes;
 import java.io.Closeable;
 import java.util.Map;
 import java.util.Objects;
@@ -74,9 +74,9 @@ public OpenTelemetryService(String clusterName,
         sdkBuilder.addPropertiesSupplier(() -> Map.of(
                 OTEL_SDK_DISABLED_KEY, "true",
                 // Cardinality limit includes the overflow attribute set, so we need to add 1.
-                "otel.experimental.metrics.cardinality.limit", Integer.toString(MAX_CARDINALITY_LIMIT + 1),
+                "otel.java.metrics.cardinality.limit", Integer.toString(MAX_CARDINALITY_LIMIT + 1),
                 // Reduce number of allocations by using reusable data mode.
-                "otel.java.experimental.exporter.memory_mode", MemoryMode.REUSABLE_DATA.name()
+                "otel.java.exporter.memory_mode", MemoryMode.REUSABLE_DATA.name()
         ));
 
         sdkBuilder.addResourceCustomizer(
@@ -87,13 +87,13 @@ public OpenTelemetryService(String clusterName,
                         resourceBuilder.put(OpenTelemetryAttributes.PULSAR_CLUSTER, clusterName);
                     }
                     if (StringUtils.isNotBlank(serviceName)
-                            && Objects.equals(Resource.getDefault().getAttribute(ResourceAttributes.SERVICE_NAME),
-                                              resource.getAttribute(ResourceAttributes.SERVICE_NAME))) {
-                        resourceBuilder.put(ResourceAttributes.SERVICE_NAME, serviceName);
+                            && Objects.equals(Resource.getDefault().getAttribute(ServiceAttributes.SERVICE_NAME),
+                                              resource.getAttribute(ServiceAttributes.SERVICE_NAME))) {
+                        resourceBuilder.put(ServiceAttributes.SERVICE_NAME, serviceName);
                     }
                     if (StringUtils.isNotBlank(serviceVersion)
-                            && resource.getAttribute(ResourceAttributes.SERVICE_VERSION) == null) {
-                        resourceBuilder.put(ResourceAttributes.SERVICE_VERSION, serviceVersion);
+                            && resource.getAttribute(ServiceAttributes.SERVICE_VERSION) == null) {
+                        resourceBuilder.put(ServiceAttributes.SERVICE_VERSION, serviceVersion);
                     }
                     return resource.merge(resourceBuilder.build());
                 });
@@ -123,7 +123,7 @@ public OpenTelemetryService(String clusterName,
                 // disable JFR based telemetry and use only JMX telemetry
                 .disableAllFeatures()
                 // enable experimental JMX telemetry in addition
-                .enableExperimentalJmxTelemetry()
+                .emitExperimentalTelemetry()
                 .build());
     }
 

pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java

@@ -19,7 +19,6 @@
 package org.apache.pulsar.opentelemetry;
 
 import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
-import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.common.Attributes;
 import io.opentelemetry.api.metrics.LongCounterBuilder;
@@ -29,7 +28,7 @@
 import io.opentelemetry.sdk.common.InstrumentationScopeInfo;
 import io.opentelemetry.sdk.metrics.export.MetricReader;
 import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader;
-import io.opentelemetry.semconv.ResourceAttributes;
+import io.opentelemetry.semconv.ServiceAttributes;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -38,7 +37,6 @@
 import lombok.Cleanup;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient;
-import org.assertj.core.api.AbstractCharSequenceAssert;
 import org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
@@ -98,7 +96,7 @@ public void testClusterNameCannotBeEmpty() {
     }
 
     @Test
-    public void testResourceAttributesAreSet() throws Exception {
+    public void testServiceAttributesAreSet() throws Exception {
         @Cleanup
         var reader = InMemoryMetricReader.create();
 
@@ -116,9 +114,8 @@ public void testResourceAttributesAreSet() throws Exception {
             .allSatisfy(metric -> assertThat(metric)
                 .hasResourceSatisfying(resource -> resource
                     .hasAttribute(OpenTelemetryAttributes.PULSAR_CLUSTER, "testServiceNameAndVersion")
-                    .hasAttribute(ResourceAttributes.SERVICE_NAME, "openTelemetryServiceTestService")
-                    .hasAttribute(ResourceAttributes.SERVICE_VERSION, "1.0.0")
-                    .hasAttribute(satisfies(ResourceAttributes.HOST_NAME, AbstractCharSequenceAssert::isNotBlank))));
+                    .hasAttribute(ServiceAttributes.SERVICE_NAME, "openTelemetryServiceTestService")
+                    .hasAttribute(ServiceAttributes.SERVICE_VERSION, "1.0.0")));
     }
 
     @Test
@@ -230,7 +227,7 @@ public void testJvmRuntimeMetrics() {
 
         // Buffer Pool Metrics
         // Replaces jvm_buffer_pool_used_bytes
-        assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.buffer.memory.usage"));
+        assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.buffer.memory.used"));
         // Replaces jvm_buffer_pool_capacity_bytes
         assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.buffer.memory.limit"));
         // Replaces jvm_buffer_pool_used_buffers