Update prom client with support for UTF-8

dependencyManagement/build.gradle.kts

@@ -15,7 +15,7 @@ val jmhVersion = "1.37"
 val mockitoVersion = "4.11.0"
 val slf4jVersion = "2.0.17"
 val opencensusVersion = "0.31.1"
-val prometheusServerVersion = "1.3.10"
+val prometheusServerVersion = "1.4.1"
 val armeriaVersion = "1.33.1"
 val junitVersion = "5.13.4"
 val okhttpVersion = "5.1.0"

exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverter.java

@@ -5,6 +5,7 @@
 
 package io.opentelemetry.exporter.prometheus;
 
+import static io.prometheus.metrics.model.snapshots.PrometheusNaming.prometheusName;
 import static io.prometheus.metrics.model.snapshots.PrometheusNaming.sanitizeLabelName;
 import static io.prometheus.metrics.model.snapshots.PrometheusNaming.sanitizeMetricName;
 import static java.util.Objects.requireNonNull;
@@ -91,6 +92,8 @@ final class Otel2PrometheusConverter {
    */
   private final Map<Attributes, List<AttributeKey<?>>> resourceAttributesToAllowedKeysCache;
 
+  private final boolean utf8SupportEnabled;
+
   /**
    * Constructor with feature flag parameter.
    *
@@ -100,13 +103,16 @@ final class Otel2PrometheusConverter {
    *     matching this predicate will be added as labels on each exported metric
    */
   Otel2PrometheusConverter(
-      boolean otelScopeEnabled, @Nullable Predicate<String> allowedResourceAttributesFilter) {
+      boolean otelScopeEnabled,
+      @Nullable Predicate<String> allowedResourceAttributesFilter,
+      boolean utf8SupportEnabled) {
     this.otelScopeEnabled = otelScopeEnabled;
     this.allowedResourceAttributesFilter = allowedResourceAttributesFilter;
     this.resourceAttributesToAllowedKeysCache =
         allowedResourceAttributesFilter != null
             ? new ConcurrentHashMap<>()
             : Collections.emptyMap();
+    this.utf8SupportEnabled = utf8SupportEnabled;
   }
 
   MetricSnapshots convert(@Nullable Collection<MetricData> metricDataCollection) {
@@ -457,8 +463,8 @@ private InfoSnapshot makeScopeInfo(Set<InstrumentationScopeInfo> scopes) {
    * Convert OpenTelemetry attributes to Prometheus labels.
    *
    * @param resource optional resource (attributes) to be converted.
-   * @param scope will be converted to {@code otel_scope_*} labels if {@code otelScopeEnabled} is
-   *     {@code true}.
+   * @param scope that will be converted to {@code otel_scope_*} labels if {@code otelScopeEnabled}
+   *     is {@code true}.
    * @param attributes the attributes to be converted.
    * @param additionalAttributes optional list of key/value pairs, may be empty.
    */
@@ -548,14 +554,20 @@ private List<AttributeKey<?>> filterAllowedResourceAttributeKeys(@Nullable Resou
     return allowedAttributeKeys;
   }
 
-  private static MetricMetadata convertMetadata(MetricData metricData) {
-    String name = sanitizeMetricName(metricData.getName());
+  private MetricMetadata convertMetadata(MetricData metricData) {
+    String name = metricData.getName();
+    if (!utf8SupportEnabled) {
+      name = prometheusName(name);
+    }
+    name = sanitizeMetricName(name);
+
     String help = metricData.getDescription();
     Unit unit = PrometheusUnitsHelper.convertUnit(metricData.getUnit());
     if (unit != null && !name.endsWith(unit.toString())) {
       name = name + "_" + unit;
     }
-    // Repeated __ are not allowed according to spec, although this is allowed in prometheus
+    // Repeated __ are discouraged according to spec, although this is allowed in prometheus, see
+    // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/compatibility/prometheus_and_openmetrics.md#metric-metadata-1
     while (name.contains("__")) {
       name = name.replace("__", "_");
     }

exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/PrometheusHttpServer.java

@@ -43,6 +43,7 @@ public final class PrometheusHttpServer implements MetricReader {
   private final String host;
   private final int port;
   private final boolean otelScopeEnabled;
+  private final boolean utf8SupportEnabled;
   @Nullable private final Predicate<String> allowedResourceAttributesFilter;
   private final MemoryMode memoryMode;
   private final DefaultAggregationSelector defaultAggregationSelector;
@@ -73,6 +74,7 @@ public static PrometheusHttpServerBuilder builder() {
       @Nullable ExecutorService executor,
       PrometheusRegistry prometheusRegistry,
       boolean otelScopeEnabled,
+      boolean utf8SupportEnabled,
       @Nullable Predicate<String> allowedResourceAttributesFilter,
       MemoryMode memoryMode,
       @Nullable HttpHandler defaultHandler,
@@ -81,12 +83,14 @@ public static PrometheusHttpServerBuilder builder() {
     this.host = host;
     this.port = port;
     this.otelScopeEnabled = otelScopeEnabled;
+    this.utf8SupportEnabled = utf8SupportEnabled;
     this.allowedResourceAttributesFilter = allowedResourceAttributesFilter;
     this.memoryMode = memoryMode;
     this.defaultAggregationSelector = defaultAggregationSelector;
     this.builder = builder;
     this.prometheusMetricReader =
-        new PrometheusMetricReader(otelScopeEnabled, allowedResourceAttributesFilter);
+        new PrometheusMetricReader(
+            otelScopeEnabled, allowedResourceAttributesFilter, utf8SupportEnabled);
     this.prometheusRegistry = prometheusRegistry;
     prometheusRegistry.register(prometheusMetricReader);
     // When memory mode is REUSABLE_DATA, concurrent reads lead to data corruption. To prevent this,
@@ -172,6 +176,7 @@ public String toString() {
     joiner.add("host=" + host);
     joiner.add("port=" + port);
     joiner.add("otelScopeEnabled=" + otelScopeEnabled);
+    joiner.add("utf8SupportEnabled=" + utf8SupportEnabled);
     joiner.add("allowedResourceAttributesFilter=" + allowedResourceAttributesFilter);
     joiner.add("memoryMode=" + memoryMode);
     joiner.add(

exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/PrometheusHttpServerBuilder.java

@@ -31,6 +31,7 @@ public final class PrometheusHttpServerBuilder {
   private int port = DEFAULT_PORT;
   private PrometheusRegistry prometheusRegistry = new PrometheusRegistry();
   private boolean otelScopeEnabled = true;
+  private boolean utf8SupportEnabled = false;
   @Nullable private Predicate<String> allowedResourceAttributesFilter;
   @Nullable private ExecutorService executor;
   private MemoryMode memoryMode = DEFAULT_MEMORY_MODE;
@@ -46,6 +47,7 @@ public final class PrometheusHttpServerBuilder {
     this.port = builder.port;
     this.prometheusRegistry = builder.prometheusRegistry;
     this.otelScopeEnabled = builder.otelScopeEnabled;
+    this.utf8SupportEnabled = builder.utf8SupportEnabled;
     this.allowedResourceAttributesFilter = builder.allowedResourceAttributesFilter;
     this.executor = builder.executor;
     this.memoryMode = builder.memoryMode;
@@ -90,6 +92,30 @@ public PrometheusHttpServerBuilder setOtelScopeEnabled(boolean otelScopeEnabled)
     return this;
   }
 
+  /**
+   * Set if UTF-8 support is enabled.
+   *
+   * <p>If set to {@code true}, the exporter will pass metric names and labels unchanged to the
+   * prometheus client library, which supports UTF-8.
+   *
+   * <p>UTF-8 will only be seen in the exported metrics if the prometheus server <a
+   * href="https://prometheus.github.io/client_java/exporters/unicode/">signals support for
+   * UTF-8</a>
+   *
+   * <p>Therefore, it's safe to always set this setting to {@code true} if you're not affected by
+   * following change in behavior:
+   *
+   * <p>If set to {@code true}, multiple non-legacy characters (e.g. <code>%%</code>) in a row will
+   * not be replaced with a single underscore as recommended in the <a href=
+   * "https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/compatibility/prometheus_and_openmetrics.md#metric-metadata-1">Prometheus
+   * conversion specification</a>.
+   */
+  @SuppressWarnings("UnusedReturnValue")
+  public PrometheusHttpServerBuilder setUtf8SupportEnabled(boolean utf8SupportEnabled) {
+    this.utf8SupportEnabled = utf8SupportEnabled;
+    return this;
+  }
+
   /**
    * Set if the resource attributes should be added as labels on each exported metric.
    *
@@ -177,6 +203,7 @@ public PrometheusHttpServer build() {
         executor,
         prometheusRegistry,
         otelScopeEnabled,
+        utf8SupportEnabled,
         allowedResourceAttributesFilter,
         memoryMode,
         defaultHandler,

exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/PrometheusMetricReader.java

@@ -29,11 +29,14 @@ public class PrometheusMetricReader implements MetricReader, MultiCollector {
   private final Otel2PrometheusConverter converter;
 
   // TODO: refactor to public static create or builder pattern to align with project style
-  /** See {@link Otel2PrometheusConverter#Otel2PrometheusConverter(boolean, Predicate)}. */
+  /** See {@link Otel2PrometheusConverter#Otel2PrometheusConverter(boolean, Predicate, boolean)}. */
   public PrometheusMetricReader(
-      boolean otelScopeEnabled, @Nullable Predicate<String> allowedResourceAttributesFilter) {
+      boolean otelScopeEnabled,
+      @Nullable Predicate<String> allowedResourceAttributesFilter,
+      boolean utf8SupportEnabled) {
     this.converter =
-        new Otel2PrometheusConverter(otelScopeEnabled, allowedResourceAttributesFilter);
+        new Otel2PrometheusConverter(
+            otelScopeEnabled, allowedResourceAttributesFilter, utf8SupportEnabled);
   }
 
   @Override

exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/internal/PrometheusComponentProvider.java

@@ -33,7 +33,9 @@ public String getName() {
 
   @Override
   public MetricReader create(DeclarativeConfigProperties config) {
-    PrometheusHttpServerBuilder prometheusBuilder = PrometheusHttpServer.builder();
+    PrometheusHttpServerBuilder prometheusBuilder =
+        PrometheusHttpServer.builder()
+            .setUtf8SupportEnabled(true); // we can accept a breaking change in declarative config
 
     Integer port = config.getInt("port");
     if (port != null) {

exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/internal/PrometheusMetricReaderProvider.java

@@ -33,6 +33,9 @@ public MetricReader createMetricReader(ConfigProperties config) {
       prometheusBuilder.setHost(host);
     }
 
+    prometheusBuilder.setUtf8SupportEnabled(
+        config.getBoolean("otel.exporter.prometheus.utf8", false));
+
     ExporterBuilderUtil.configureExporterMemoryMode(config, prometheusBuilder::setMemoryMode);
 
     String defaultHistogramAggregation =

exporters/prometheus/src/test/java/io/opentelemetry/exporter/prometheus/CollectorIntegrationTest.java

@@ -133,25 +133,24 @@ void endToEnd() {
     assertThat(resourceMetrics.getResource().getAttributesList())
         .containsExactlyInAnyOrder(
             // Resource attributes derived from the prometheus scrape config
-            stringKeyValue("service.name", "app"),
             stringKeyValue("service.instance.id", "host.testcontainers.internal:" + prometheusPort),
             stringKeyValue("server.address", "host.testcontainers.internal"),
             stringKeyValue("server.port", String.valueOf(prometheusPort)),
             stringKeyValue("url.scheme", "http"),
             // Resource attributes from the metric SDK resource translated to target_info
             stringKeyValue(
-                "service_name",
+                "service.name",
                 Objects.requireNonNull(resource.getAttributes().get(stringKey("service.name")))),
             stringKeyValue(
-                "telemetry_sdk_name",
+                "telemetry.sdk.name",
                 Objects.requireNonNull(
                     resource.getAttributes().get(stringKey("telemetry.sdk.name")))),
             stringKeyValue(
-                "telemetry_sdk_language",
+                "telemetry.sdk.language",
                 Objects.requireNonNull(
                     resource.getAttributes().get(stringKey("telemetry.sdk.language")))),
             stringKeyValue(
-                "telemetry_sdk_version",
+                "telemetry.sdk.version",
                 Objects.requireNonNull(
                     resource.getAttributes().get(stringKey("telemetry.sdk.version")))));
 

exporters/prometheus/src/test/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverterTest.java

@@ -37,6 +37,7 @@
 import io.opentelemetry.sdk.metrics.internal.data.ImmutableSummaryData;
 import io.opentelemetry.sdk.metrics.internal.data.ImmutableSummaryPointData;
 import io.opentelemetry.sdk.resources.Resource;
+import io.prometheus.metrics.config.EscapingScheme;
 import io.prometheus.metrics.expositionformats.ExpositionFormats;
 import io.prometheus.metrics.model.snapshots.CounterSnapshot;
 import io.prometheus.metrics.model.snapshots.Labels;
@@ -68,24 +69,77 @@ class Otel2PrometheusConverterTest {
           "(.|\\n)*# HELP (?<help>.*)\n# TYPE (?<type>.*)\n(?<metricName>.*)\\{"
               + "otel_scope_foo=\"bar\",otel_scope_name=\"scope\","
               + "otel_scope_schema_url=\"schemaUrl\",otel_scope_version=\"version\"}(.|\\n)*");
+  private static final Pattern ESCAPE_PATTERN =
+      Pattern.compile(
+          "(.|\\n)*# HELP (?<help>.*)\n# TYPE (?<type>.*)\n\\{\"(?<metricName>.*)\","
+              + "otel_scope_foo=\"bar\",otel_scope_name=\"scope\","
+              + "otel_scope_schema_url=\"schemaUrl\",otel_scope_version=\"version\"}(.|\\n)*");
   private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
   private final Otel2PrometheusConverter converter =
-      new Otel2PrometheusConverter(true, /* allowedResourceAttributesFilter= */ null);
+      new Otel2PrometheusConverter(
+          true, /* allowedResourceAttributesFilter= */ null, /* utf8SupportEnabled */ true);
 
   @ParameterizedTest
   @MethodSource("metricMetadataArgs")
   void metricMetadata(
       MetricData metricData, String expectedType, String expectedHelp, String expectedMetricName)
       throws IOException {
+    assertMetricData(
+        metricData,
+        expectedType,
+        expectedHelp,
+        expectedMetricName,
+        new Otel2PrometheusConverter(
+            true, /* allowedResourceAttributesFilter= */ null, /* utf8SupportEnabled */ false),
+        EscapingScheme.UNDERSCORE_ESCAPING,
+        PATTERN);
+  }
+
+  @Test
+  void metricMetadataUtf8() throws IOException {
+    // all UTF-8 chars are accepted as is
+    // repeated "_" are collapsed, but 2 λ are not collapsed to a single "_"
+    // In a real application, the escaping scheme is passed using the "escaping" header when
+    // scraping the metrics
+
+    MetricData metricData = createSampleMetricData("λλbe__happy", "1", MetricDataType.LONG_GAUGE);
+    assertMetricData(
+        metricData,
+        "__be_happy_ratio gauge",
+        "__be_happy_ratio description",
+        "__be_happy_ratio",
+        converter,
+        EscapingScheme.UNDERSCORE_ESCAPING,
+        PATTERN);
+
+    assertMetricData(
+        metricData,
+        "\"λλbe_happy_ratio\" gauge",
+        "\"λλbe_happy_ratio\" description",
+        "λλbe_happy_ratio",
+        converter,
+        EscapingScheme.ALLOW_UTF8,
+        ESCAPE_PATTERN);
+  }
+
+  private static void assertMetricData(
+      MetricData metricData,
+      String expectedType,
+      String expectedHelp,
+      String expectedMetricName,
+      Otel2PrometheusConverter converter,
+      EscapingScheme escapingScheme,
+      Pattern pattern)
+      throws IOException {
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     MetricSnapshots snapshots = converter.convert(Collections.singletonList(metricData));
-    ExpositionFormats.init().getPrometheusTextFormatWriter().write(out, snapshots);
+    ExpositionFormats.init().getPrometheusTextFormatWriter().write(out, snapshots, escapingScheme);
     String expositionFormat = new String(out.toByteArray(), StandardCharsets.UTF_8);
 
     assertThat(expositionFormat)
         .matchesSatisfying(
-            PATTERN,
+            pattern,
             matcher -> {
               assertThat(matcher.group("help")).isEqualTo(expectedHelp);
               assertThat(matcher.group("type")).isEqualTo(expectedType);
@@ -138,12 +192,13 @@ private static Stream<Arguments> metricMetadataArgs() {
             "metric_name_2 summary",
             "metric_name_2 description",
             "metric_name_2_count"),
-        // unsupported characters are translated to "_", repeated "_" are dropped
+        // unsupported characters are translated to "_", repeated "_" are collapsed if
+        // the original name had consecutive "_"
         Arguments.of(
-            createSampleMetricData("s%%ple", "%/min", MetricDataType.SUMMARY),
-            "s_ple_percent_per_minute summary",
-            "s_ple_percent_per_minute description",
-            "s_ple_percent_per_minute_count"),
+            createSampleMetricData("s%%p__le", "%/min", MetricDataType.SUMMARY),
+            "s_p_le_percent_per_minute summary",
+            "s_p_le_percent_per_minute description",
+            "s_p_le_percent_per_minute_count"),
         // metric unit is not appended if the name already contains the unit
         Arguments.of(
             createSampleMetricData("metric_name_total", "total", MetricDataType.LONG_SUM),
@@ -201,7 +256,8 @@ void resourceAttributesAddition(
       throws IOException {
 
     Otel2PrometheusConverter converter =
-        new Otel2PrometheusConverter(true, allowedResourceAttributesFilter);
+        new Otel2PrometheusConverter(
+            true, allowedResourceAttributesFilter, /* utf8SupportEnabled */ true);
 
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     MetricSnapshots snapshots = converter.convert(Collections.singletonList(metricData));
@@ -501,7 +557,10 @@ void validateCacheIsBounded() {
         };
 
     Otel2PrometheusConverter otel2PrometheusConverter =
-        new Otel2PrometheusConverter(true, /* allowedResourceAttributesFilter= */ countPredicate);
+        new Otel2PrometheusConverter(
+            true,
+            /* allowedResourceAttributesFilter= */ countPredicate,
+            /* utf8SupportEnabled */ true);
 
     // Create 20 different metric data objects with 2 different resource attributes;
     Resource resource1 = Resource.builder().put("cluster", "cluster1").build();

exporters/prometheus/src/test/java/io/opentelemetry/exporter/prometheus/PrometheusHttpServerTest.java

@@ -46,7 +46,7 @@
 import io.opentelemetry.sdk.resources.Resource;
 import io.prometheus.metrics.exporter.httpserver.HTTPServer;
 import io.prometheus.metrics.exporter.httpserver.MetricsHandler;
-import io.prometheus.metrics.expositionformats.generated.com_google_protobuf_4_31_1.Metrics;
+import io.prometheus.metrics.expositionformats.generated.com_google_protobuf_4_32_0.Metrics;
 import io.prometheus.metrics.model.registry.PrometheusRegistry;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -414,6 +414,7 @@ void stringRepresentation() {
                 + "host=localhost,"
                 + "port=0,"
                 + "otelScopeEnabled=true,"
+                + "utf8SupportEnabled=false,"
                 + "allowedResourceAttributesFilter=null,"
                 + "memoryMode=REUSABLE_DATA,"
                 + "defaultAggregationSelector=DefaultAggregationSelector{COUNTER=default, UP_DOWN_COUNTER=default, HISTOGRAM=default, OBSERVABLE_COUNTER=default, OBSERVABLE_UP_DOWN_COUNTER=default, OBSERVABLE_GAUGE=default, GAUGE=default}"