Stu elastic otel meter api

TRACING.md

@@ -158,6 +158,6 @@ explicitly opening a scope via the `Tracer`.
 [otel]: https://opentelemetry.io/
 [thread-context]: ./server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
 [w3c]: https://www.w3.org/TR/trace-context/
-[tracing]: ./server/src/main/java/org/elasticsearch/tracing
+[tracing]: ./server/src/main/java/org/elasticsearch/telemetry
 [agent-config]: https://www.elastic.co/guide/en/apm/agent/java/master/configuration.html
 [agent]: https://www.elastic.co/guide/en/apm/agent/java/current/index.html

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java

@@ -50,10 +50,12 @@ class APMJvmOptions {
         // by the agent. Don't disable writing to a log file, as the agent will then
         // require extra Security Manager permissions when it tries to do something
         // else, and it's just painful.
-        "log_file", "_AGENT_HOME_/../../logs/apm.log",
-
+        "log_file", "/Users/przemyslawgomulka/workspace/pgomulka/apm.log",
+        "log_level", "debug",
         // ES does not use auto-instrumentation.
-        "instrument", "false"
+        "instrument", "false",
+        "experimental", "true",
+        "enable_experimental_instrumentations", "true"
         );
 
     /**
@@ -82,7 +84,7 @@ class APMJvmOptions {
         // is doing, leave this value alone.
         "log_level", "error",
         "application_packages", "org.elasticsearch,org.apache.lucene",
-        "metrics_interval", "120s",
+        "metrics_interval", "5s",
         "breakdown_metrics", "false",
         "central_config", "false"
         );
@@ -316,9 +318,7 @@ static Path findAgentJar(String installDir) throws IOException, UserException {
         }
 
         try (var apmStream = Files.list(apmModule)) {
-            final List<Path> paths = apmStream.filter(
-                path -> path.getFileName().toString().matches("elastic-apm-agent-\\d+\\.\\d+\\.\\d+\\.jar")
-            ).toList();
+            final List<Path> paths = apmStream.filter(path -> path.getFileName().toString().matches("elastic-apm-agent-.*.jar")).toList();
 
             if (paths.size() > 1) {
                 throw new UserException(

gradle/verification-metadata.xml

@@ -74,6 +74,11 @@
             <sha256 value="3be90268557494f88fe252caf6358d4107aa9349852b8704b1e0ff4831cb74d6" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="co.elastic.apm" name="elastic-apm-agent" version="1.37.0">
+         <artifact name="elastic-apm-agent-1.37.0.jar">
+            <sha256 value="10496390d1dcc7c558ae63e115beff9477157e3d219fce686ef5ca59c744dfea" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="co.elastic.logging" name="ecs-logging-core" version="1.2.0">
          <artifact name="ecs-logging-core-1.2.0.jar">
             <sha256 value="0d6318af29848ea588b1c244834e3e762315c68de2bcfc9f1adf3f0633d22d37" origin="Generated by Gradle"/>

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java

@@ -131,7 +131,7 @@
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.synonyms.SynonymsManagementAPIService;
-import org.elasticsearch.telemetry.tracing.Tracer;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -168,7 +168,7 @@ public Collection<Object> createComponents(
         NamedWriteableRegistry namedWriteableRegistry,
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
-        Tracer tracer,
+        TelemetryProvider telemetryProvider,
         AllocationService allocationService,
         IndicesService indicesService
     ) {

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java

@@ -28,7 +28,7 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.telemetry.tracing.Tracer;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -68,7 +68,22 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
         };
         Client client = new MockClient(Settings.EMPTY, null);
         CommonAnalysisPlugin plugin = new CommonAnalysisPlugin();
-        plugin.createComponents(client, null, null, null, scriptService, null, null, null, null, null, null, Tracer.NOOP, null, null);
+        plugin.createComponents(
+            client,
+            null,
+            null,
+            null,
+            scriptService,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            TelemetryProvider.NOOP,
+            null,
+            null
+        );
         AnalysisModule module = new AnalysisModule(
             TestEnvironment.newEnvironment(settings),
             Collections.singletonList(plugin),

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java

@@ -28,7 +28,7 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.telemetry.tracing.Tracer;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -68,7 +68,22 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
         };
         Client client = new MockClient(Settings.EMPTY, null);
         CommonAnalysisPlugin plugin = new CommonAnalysisPlugin();
-        plugin.createComponents(client, null, null, null, scriptService, null, null, null, null, null, null, Tracer.NOOP, null, null);
+        plugin.createComponents(
+            client,
+            null,
+            null,
+            null,
+            scriptService,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            TelemetryProvider.NOOP,
+            null,
+            null
+        );
         AnalysisModule module = new AnalysisModule(
             TestEnvironment.newEnvironment(settings),
             Collections.singletonList(plugin),

modules/apm/build.gradle

@@ -14,11 +14,19 @@ esplugin {
 
 def otelVersion = '1.17.0'
 
+repositories {
+  maven {
+    name "sonatype-nexus-snapshots"
+    url "https://oss.sonatype.org/content/repositories/snapshots"
+  }
+}
 dependencies {
   implementation "io.opentelemetry:opentelemetry-api:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-context:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-semconv:${otelVersion}-alpha"
-  runtimeOnly    "co.elastic.apm:elastic-apm-agent:1.36.0"
+  implementation "org.apache.logging.log4j:log4j-api:${versions.log4j}"
+
+  runtimeOnly    "co.elastic.apm:elastic-apm-agent:1.42.1-SNAPSHOT"
 }
 
 tasks.named("dependencyLicenses").configure {

modules/apm/src/main/java/module-info.java

@@ -16,4 +16,5 @@
     requires io.opentelemetry.api;
 
     exports org.elasticsearch.telemetry.apm;
+    exports org.elasticsearch.telemetry.apm.internal.metrics;
 }

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java

@@ -21,12 +21,14 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.TracerPlugin;
+import org.elasticsearch.plugins.TelemetryPlugin;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.telemetry.apm.settings.APMAgentSettings;
-import org.elasticsearch.telemetry.apm.tracing.APMTracer;
-import org.elasticsearch.telemetry.tracing.Tracer;
+import org.elasticsearch.telemetry.TelemetryProvider;
+import org.elasticsearch.telemetry.apm.internal.APMAgentSettings;
+import org.elasticsearch.telemetry.apm.internal.APMTelemetryProvider;
+import org.elasticsearch.telemetry.apm.internal.metrics.APMMetric;
+import org.elasticsearch.telemetry.apm.internal.tracing.APMTracer;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -55,19 +57,19 @@
  * be passed via system properties to the Java agent, which periodically checks for changes
  * and applies the new settings values, provided those settings can be dynamically updated.
  */
-public class APM extends Plugin implements NetworkPlugin, TracerPlugin {
-    private final SetOnce<APMTracer> tracer = new SetOnce<>();
+public class APM extends Plugin implements NetworkPlugin, TelemetryPlugin {
+    private final SetOnce<APMTelemetryProvider> telemetryProvider = new SetOnce<>();
     private final Settings settings;
 
     public APM(Settings settings) {
         this.settings = settings;
     }
 
     @Override
-    public Tracer getTracer(Settings settings) {
-        final APMTracer apmTracer = new APMTracer(settings);
-        tracer.set(apmTracer);
-        return apmTracer;
+    public TelemetryProvider getTelemetryProvider(Settings settings) {
+        final APMTelemetryProvider apmTelemetryProvider = new APMTelemetryProvider(settings);
+        telemetryProvider.set(apmTelemetryProvider);
+        return apmTelemetryProvider;
     }
 
     @Override
@@ -83,20 +85,22 @@ public Collection<Object> createComponents(
         NamedWriteableRegistry namedWriteableRegistry,
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
-        Tracer unused,
+        TelemetryProvider unused,
         AllocationService allocationService,
         IndicesService indicesService
     ) {
-        final APMTracer apmTracer = tracer.get();
+        final APMTracer apmTracer = telemetryProvider.get().getTracer();
 
         apmTracer.setClusterName(clusterService.getClusterName().value());
         apmTracer.setNodeName(clusterService.getNodeName());
 
         final APMAgentSettings apmAgentSettings = new APMAgentSettings();
         apmAgentSettings.syncAgentSystemProperties(settings);
-        apmAgentSettings.addClusterSettingsListeners(clusterService, apmTracer);
+        apmAgentSettings.addClusterSettingsListeners(clusterService, telemetryProvider.get());
 
-        return List.of(apmTracer);
+        final APMMetric apmMetric = telemetryProvider.get().getMetric();
+
+        return List.of(apmTracer, apmMetric);
     }
 
     @Override

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/settings/APMAgentSettings.java → modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.telemetry.apm.settings;
+package org.elasticsearch.telemetry.apm.internal;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -17,7 +17,8 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.telemetry.apm.tracing.APMTracer;
+import org.elasticsearch.telemetry.apm.internal.metrics.APMMetric;
+import org.elasticsearch.telemetry.apm.internal.tracing.APMTracer;
 
 import java.security.AccessController;
 import java.security.PrivilegedAction;
@@ -40,12 +41,21 @@ public class APMAgentSettings {
      * Sensible defaults that Elasticsearch configures. This cannot be done via the APM agent
      * config file, as then their values could not be overridden dynamically via system properties.
      */
-    static Map<String, String> APM_AGENT_DEFAULT_SETTINGS = Map.of("transaction_sample_rate", "0.2");
+    static Map<String, String> APM_AGENT_DEFAULT_SETTINGS = Map.of(
+        "transaction_sample_rate",
+        "0.2",
+        "enable_experimental_instrumentations",
+        "true"
+    );
 
-    public void addClusterSettingsListeners(ClusterService clusterService, APMTracer apmTracer) {
+    public void addClusterSettingsListeners(ClusterService clusterService, APMTelemetryProvider apmTelemetryProvider) {
         final ClusterSettings clusterSettings = clusterService.getClusterSettings();
+        final APMTracer apmTracer = apmTelemetryProvider.getTracer();
+        final APMMetric apmMetric = apmTelemetryProvider.getMetric();
+
         clusterSettings.addSettingsUpdateConsumer(APM_ENABLED_SETTING, enabled -> {
             apmTracer.setEnabled(enabled);
+            apmMetric.setEnabled(enabled);
             // The agent records data other than spans, e.g. JVM metrics, so we toggle this setting in order to
             // minimise its impact to a running Elasticsearch.
             this.setAgentSetting("recording", Boolean.toString(enabled));

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMTelemetryProvider.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.telemetry.apm.internal;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.telemetry.TelemetryProvider;
+import org.elasticsearch.telemetry.apm.internal.metrics.APMMetric;
+import org.elasticsearch.telemetry.apm.internal.tracing.APMTracer;
+
+public class APMTelemetryProvider implements TelemetryProvider {
+    private final Settings settings;
+    private final APMTracer apmTracer;
+    private final APMMetric apmMetric;
+
+    public APMTelemetryProvider(Settings settings) {
+        this.settings = settings;
+        apmTracer = new APMTracer(settings);
+        apmMetric = new APMMetric(settings);
+    }
+
+    @Override
+    public APMTracer getTracer() {
+        return apmTracer;
+    }
+
+    @Override
+    public APMMetric getMetric() {
+        return apmMetric;
+    }
+}