Add metrics OTLP endpoint skeleton

gradle/verification-metadata.xml

@@ -1751,6 +1751,16 @@
             <sha256 value="2eaaac5f268b135f0e11dd30637d71df5751a0bb7ed6268659be57104d63122b" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="io.opentelemetry" name="opentelemetry-api" version="1.53.0">
+         <artifact name="opentelemetry-api-1.53.0.jar">
+            <sha256 value="1991903b9fc76b27f1e6a70a5a97131668fb5f3ac9026178c450d510cbb1bef2" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry" name="opentelemetry-common" version="1.53.0">
+         <artifact name="opentelemetry-common-1.53.0.jar">
+            <sha256 value="73a5181dd07e72c4312fadafc8328ad1046a32c050030e3a7b8c16113daad359" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="io.opentelemetry" name="opentelemetry-context" version="1.31.0">
          <artifact name="opentelemetry-context-1.31.0.jar">
             <sha256 value="664896a5c34bcda20c95c8f45198a95e8f97a1cd5e5c2923978f42dddada787d" origin="Generated by Gradle"/>
@@ -1766,11 +1776,61 @@
             <sha256 value="76f9dfe1a6f74d5081e07bde1f7cb9a06879d317ec0ae0f61dd8fb2be9afad4f" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="io.opentelemetry" name="opentelemetry-context" version="1.53.0">
+         <artifact name="opentelemetry-context-1.53.0.jar">
+            <sha256 value="88a780c5bb7b51e7c07071d61403e34f13277d092bf441a5cb49cf2ce9c756e9" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry" name="opentelemetry-exporter-common" version="1.53.0">
+         <artifact name="opentelemetry-exporter-common-1.53.0.jar">
+            <sha256 value="d7f093f987547c9c2c2caa28baf1254507c59b4dbf97b49fee5c9cd1ceb6f8d5" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry" name="opentelemetry-exporter-otlp" version="1.53.0">
+         <artifact name="opentelemetry-exporter-otlp-1.53.0.jar">
+            <sha256 value="0ce88bf35577894ee185f718f1b981938bbfc8981f3b7fcecbfd7b7c375bc236" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry" name="opentelemetry-exporter-otlp-common" version="1.53.0">
+         <artifact name="opentelemetry-exporter-otlp-common-1.53.0.jar">
+            <sha256 value="32bcf0e855110b09cffbb5aac59c69d2caaa35de646e07c0a5f7f50ab42d4c1f" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry" name="opentelemetry-exporter-sender-jdk" version="1.53.0">
+         <artifact name="opentelemetry-exporter-sender-jdk-1.53.0.jar">
+            <sha256 value="d147fd99aa23be8b0ea8acbc06ffae2f091af2edfffeb44163ab79765f7e3b42" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="io.opentelemetry" name="opentelemetry-sdk" version="1.47.0">
          <artifact name="opentelemetry-sdk-1.47.0.jar">
             <sha256 value="4a09eb2ee484769973e14218a34e6da54f35955aa02b26dc5238b0c2ed6a801d" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="io.opentelemetry" name="opentelemetry-sdk" version="1.53.0">
+         <artifact name="opentelemetry-sdk-1.53.0.jar">
+            <sha256 value="d58721063bca5d612bf1f6dccb883664210999c5aa470c149d1509672f13229c" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry" name="opentelemetry-sdk-common" version="1.53.0">
+         <artifact name="opentelemetry-sdk-common-1.53.0.jar">
+            <sha256 value="a5d68aca0920aa0dc17b9ecc5510abe4c344e66eae9a4420c992298e4712d3bc" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry" name="opentelemetry-sdk-extension-autoconfigure" version="1.53.0">
+         <artifact name="opentelemetry-sdk-extension-autoconfigure-1.53.0.jar">
+            <sha256 value="72becaccfb3d79d91c6486bf4e68f2474a1aa7742934fb675ea077e35a8dc3e1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry" name="opentelemetry-sdk-extension-autoconfigure-spi" version="1.53.0">
+         <artifact name="opentelemetry-sdk-extension-autoconfigure-spi-1.53.0.jar">
+            <sha256 value="e814e63dc2f8cbdf84574b6289eb543b89b06486d03cad1e697b06d802ce27bb" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry" name="opentelemetry-sdk-metrics" version="1.53.0">
+         <artifact name="opentelemetry-sdk-metrics-1.53.0.jar">
+            <sha256 value="3d7dbae6c03be035e7c4dbdd50442741b260be5463fa9aadca1b00af0efddee5" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="io.opentelemetry" name="opentelemetry-semconv" version="1.21.0-alpha">
          <artifact name="opentelemetry-semconv-1.21.0-alpha.jar">
             <sha256 value="4a8f41b93eec51e85fa6b48e43de6785b742316fdd9c9baf595adbce6d5de6af" origin="Generated by Gradle"/>

x-pack/plugin/otel-data/build.gradle

@@ -9,6 +9,7 @@ import com.google.protobuf.gradle.GenerateProtoTask
 plugins {
   id 'elasticsearch.internal-es-plugin'
   id 'elasticsearch.internal-yaml-rest-test'
+  id 'elasticsearch.internal-java-rest-test'
   id 'elasticsearch.internal-cluster-test'
   id('com.google.protobuf') version '0.9.5'
 }
@@ -47,10 +48,9 @@ dependencies {
   });
   opentelemetryProtobuf "open-telemetry:opentelemetry-proto:1.7.0@zip"
 
+  api project(":libs:exponential-histogram")
   compileOnly project(path: xpackModule('core'))
-  testImplementation project(path: ':x-pack:plugin:stack')
   testImplementation(testArtifact(project(xpackModule('core'))))
-  testImplementation project(':modules:data-streams')
   clusterModules project(':modules:data-streams')
   clusterModules project(':modules:ingest-common')
   clusterModules project(':modules:ingest-geoip')
@@ -66,6 +66,20 @@ dependencies {
   clusterModules project(xpackModule('wildcard'))
   clusterModules project(xpackModule('mapper-version'))
 
+  def otelVersion = "1.53.0"
+  javaRestTestImplementation "io.opentelemetry:opentelemetry-api:$otelVersion"
+  javaRestTestImplementation "io.opentelemetry:opentelemetry-common:$otelVersion"
+  javaRestTestImplementation "io.opentelemetry:opentelemetry-context:$otelVersion"
+  javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk:$otelVersion"
+  javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk-common:$otelVersion"
+  javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk-metrics:$otelVersion"
+  javaRestTestImplementation "io.opentelemetry:opentelemetry-exporter-common:$otelVersion"
+  javaRestTestImplementation "io.opentelemetry:opentelemetry-exporter-otlp:$otelVersion"
+  javaRestTestImplementation "io.opentelemetry:opentelemetry-exporter-otlp-common:$otelVersion"
+  javaRestTestImplementation "io.opentelemetry:opentelemetry-exporter-sender-jdk:$otelVersion"
+  javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:$otelVersion"
+  javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:$otelVersion"
+
   implementation "com.google.protobuf:protobuf-java:${protobufVersion}"
   // The protobuf plugin only adds a dependency for the variant relevant for the current platform.
   // Without explicitly adding all classifiers, the --write-verification-metadata task would only add the one for the current platform.
@@ -150,3 +164,7 @@ tasks.named("thirdPartyAudit").configure {
 tasks.named("licenseHeaders").configure {
   excludes << 'io/opentelemetry/proto/**/*'
 }
+
+tasks.named("javaRestTest").configure {
+  usesDefaultDistribution("Requires a bunch of xpack plugins")
+}

x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingRestIT.java

@@ -0,0 +1,147 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.action.otlp;
+
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.exporter.internal.FailedExportException;
+import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter;
+import io.opentelemetry.sdk.common.Clock;
+import io.opentelemetry.sdk.common.CompletableResultCode;
+import io.opentelemetry.sdk.common.InstrumentationScopeInfo;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.opentelemetry.sdk.metrics.data.MetricData;
+import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
+import io.opentelemetry.sdk.metrics.internal.data.ImmutableDoublePointData;
+import io.opentelemetry.sdk.metrics.internal.data.ImmutableGaugeData;
+import io.opentelemetry.sdk.metrics.internal.data.ImmutableMetricData;
+import io.opentelemetry.sdk.resources.Resource;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.Before;
+import org.junit.ClassRule;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static io.opentelemetry.api.common.AttributeKey.stringKey;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class OTLPMetricsIndexingRestIT extends ESRestTestCase {
+
+    private static final String USER = "test_admin";
+    private static final String PASS = "x-pack-test-password";
+    private static final Resource TEST_RESOURCE = Resource.create(Attributes.of(stringKey("service.name"), "elasticsearch"));
+    private static final InstrumentationScopeInfo TEST_SCOPE = InstrumentationScopeInfo.create("io.opentelemetry.example.metrics");
+    private OtlpHttpMetricExporter exporter;
+    private SdkMeterProvider meterProvider;
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .user(USER, PASS, "superuser", false)
+        .setting("xpack.security.autoconfiguration.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()));
+        return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    @Before
+    public void beforeTest() throws Exception {
+        exporter = OtlpHttpMetricExporter.builder()
+            .setEndpoint(getClusterHosts().getFirst().toURI() + "/_otlp/v1/metrics")
+            .addHeader("Authorization", basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())))
+            .build();
+        meterProvider = SdkMeterProvider.builder()
+            .registerMetricReader(
+                PeriodicMetricReader.builder(exporter)
+                    .setExecutor(Executors.newScheduledThreadPool(0))
+                    .setInterval(Duration.ofNanos(Long.MAX_VALUE))
+                    .build()
+            )
+            .build();
+        assertBusy(() -> assertOK(client().performRequest(new Request("GET", "_index_template/metrics-otel@template"))));
+        boolean otlpEndpointEnabled = false;
+        try {
+            otlpEndpointEnabled = RestStatus.isSuccessful(
+                client().performRequest(new Request("POST", "/_otlp/v1/metrics")).getStatusLine().getStatusCode()
+            );
+        } catch (Exception ignore) {}
+        assumeTrue("Requires otlp_metrics feature flag to be enabled", otlpEndpointEnabled);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        meterProvider.close();
+        super.tearDown();
+    }
+
+    public void testIngestMetricDataViaMetricExporter() throws Exception {
+        MetricData jvmMemoryMetricData = createDoubleGauge(
+            TEST_RESOURCE,
+            Attributes.empty(),
+            "jvm.memory.total",
+            Runtime.getRuntime().totalMemory(),
+            "By",
+            Clock.getDefault().now()
+        );
+
+        FailedExportException.HttpExportException exception = assertThrows(
+            FailedExportException.HttpExportException.class,
+            () -> export(List.of(jvmMemoryMetricData))
+        );
+        assertThat(exception.getResponse().statusCode(), equalTo(RestStatus.NOT_IMPLEMENTED.getStatus()));
+    }
+
+    private void export(List<MetricData> metrics) throws Exception {
+        CompletableResultCode result = exporter.export(metrics).join(10, TimeUnit.SECONDS);
+        Throwable failure = result.getFailureThrowable();
+        if (failure instanceof Exception e) {
+            throw e;
+        } else if (failure != null) {
+            throw new RuntimeException("Failed to export metrics", failure);
+        }
+        assertThat(result.isSuccess(), is(true));
+        assertOK(client().performRequest(new Request("GET", "_refresh/metrics-*")));
+    }
+
+    private static MetricData createDoubleGauge(
+        Resource resource,
+        Attributes attributes,
+        String name,
+        double value,
+        String unit,
+        long timeEpochNanos
+    ) {
+        return ImmutableMetricData.createDoubleGauge(
+            resource,
+            TEST_SCOPE,
+            name,
+            "Your description could be here.",
+            unit,
+            ImmutableGaugeData.create(List.of(ImmutableDoublePointData.create(timeEpochNanos, timeEpochNanos, attributes, value)))
+        );
+    }
+}

x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java

@@ -10,23 +10,31 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.util.FeatureFlag;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.oteldata.otlp.OTLPMetricsRestAction;
+import org.elasticsearch.xpack.oteldata.otlp.OTLPMetricsTransportAction;
 
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
 
 public class OTelPlugin extends Plugin implements ActionPlugin {
-    private static final Logger logger = LogManager.getLogger(OTelPlugin.class);
-
-    final SetOnce<OTelIndexTemplateRegistry> registry = new SetOnce<>();
-
-    private final boolean enabled;
 
     // OTEL_DATA_REGISTRY_ENABLED controls enabling the index template registry.
     //
@@ -38,10 +46,35 @@ public class OTelPlugin extends Plugin implements ActionPlugin {
         Setting.Property.Dynamic
     );
 
+    private static final Logger logger = LogManager.getLogger(OTelPlugin.class);
+
+    private static final boolean OTLP_METRICS_ENABLED = new FeatureFlag("otlp_metrics").isEnabled();
+    private final SetOnce<OTelIndexTemplateRegistry> registry = new SetOnce<>();
+    private final boolean enabled;
+
     public OTelPlugin(Settings settings) {
         this.enabled = XPackSettings.OTEL_DATA_ENABLED.get(settings);
     }
 
+    @Override
+    public Collection<RestHandler> getRestHandlers(
+        Settings settings,
+        NamedWriteableRegistry namedWriteableRegistry,
+        RestController restController,
+        ClusterSettings clusterSettings,
+        IndexScopedSettings indexScopedSettings,
+        SettingsFilter settingsFilter,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier<DiscoveryNodes> nodesInCluster,
+        Predicate<NodeFeature> clusterSupportsFeature
+    ) {
+        if (OTLP_METRICS_ENABLED) {
+            return List.of(new OTLPMetricsRestAction());
+        } else {
+            return List.of();
+        }
+    }
+
     @Override
     public Collection<?> createComponents(PluginServices services) {
         logger.info("OTel ingest plugin is {}", enabled ? "enabled" : "disabled");
@@ -55,7 +88,7 @@ public Collection<?> createComponents(PluginServices services) {
             registryInstance.setEnabled(OTEL_DATA_REGISTRY_ENABLED.get(settings));
             registryInstance.initialize();
         }
-        return Collections.emptyList();
+        return List.of();
     }
 
     @Override
@@ -67,4 +100,13 @@ public void close() {
     public List<Setting<?>> getSettings() {
         return List.of(OTEL_DATA_REGISTRY_ENABLED);
     }
+
+    @Override
+    public Collection<ActionHandler> getActions() {
+        if (OTLP_METRICS_ENABLED) {
+            return List.of(new ActionHandler(OTLPMetricsTransportAction.TYPE, OTLPMetricsTransportAction.class));
+        } else {
+            return List.of();
+        }
+    }
 }