add runtime attach (#589)

Author: SylvainJuge

agent/entrypoint/src/main/java/co/elastic/otel/agent/ElasticAgent.java

@@ -24,12 +24,34 @@
 /** Elastic agent entry point, delegates to OpenTelemetry agent */
 public class ElasticAgent {
 
-  @SuppressWarnings("unused")
+  /**
+   * Entry point for -javaagent JVM argument attach
+   *
+   * @param agentArgs agent arguments
+   * @param inst instrumentation
+   */
   public static void premain(String agentArgs, Instrumentation inst) {
     OpenTelemetryAgent.premain(agentArgs, inst);
   }
 
+  /**
+   * Entry point for runtime attach
+   *
+   * @param agentArgs agent arguments
+   * @param inst instrumentation
+   */
+  public static void agentmain(String agentArgs, Instrumentation inst) {
+    OpenTelemetryAgent.agentmain(agentArgs, inst);
+  }
+
+  /**
+   * Entry point to execute as program
+   *
+   * @param args arguments
+   */
   public static void main(String[] args) {
     OpenTelemetryAgent.main(args);
   }
+
+  private ElasticAgent() {}
 }

gradle/libs.versions.toml

@@ -35,6 +35,7 @@ gcpContribResources = { group = "io.opentelemetry.contrib", name = "opentelemetr
 contribResources = { group = "io.opentelemetry.contrib", name = "opentelemetry-resource-providers", version.ref = "opentelemetryContribAlpha" }
 contribSpanStacktrace = { group = "io.opentelemetry.contrib", name = "opentelemetry-span-stacktrace", version.ref = "opentelemetryContribAlpha" }
 contribInferredSpans = { group = "io.opentelemetry.contrib", name = "opentelemetry-inferred-spans", version.ref = "opentelemetryContribAlpha" }
+contribRuntimeAttach = { group = "io.opentelemetry.contrib", name = "opentelemetry-runtime-attach-core", version.ref = "opentelemetryContribAlpha" }
 
 opentelemetrySemconv = { group = "io.opentelemetry.semconv", name = "opentelemetry-semconv", version.ref = "opentelemetrySemconv" }
 opentelemetrySemconvIncubating = { group = "io.opentelemetry.semconv", name = "opentelemetry-semconv-incubating", version.ref = "opentelemetrySemconvAlpha" }

runtime-attach/README.md

@@ -0,0 +1,72 @@
+
+# EDOT Runtime attach
+
+This feature is currently in **Tech Preview**.
+
+Runtime attach allows to:
+
+- Allows to deploy the agent when access to JVM arguments or configuration is not possible, for example with some managed services
+- Agent deployment can be controlled by the application development team, without needing modifications to the run environment
+
+However, it also has some limitations:
+
+- It can't be used with multiple applications that share a JVM, for example with web-applications and application servers, in this case only the `-javaagent` option is supported
+- Agent update is tied to the application development and deployment lifecycle.
+- It requires minor modification of the application `main` entry point and adding one extra dependency.
+- Agent can only be attached at application start, it can't be used to attach later during application runtime.
+- Recent JVMs issue [warnings in standard error](#jvm-runtime-attach-warnings) and the feature might require explicit opt-in with JVM settings in the future 
+
+## Setup
+
+Adding runtime attach to an application is a 3-step process:
+1. add runtime attach to the application dependencies
+2. minor code modification of the application `main` method
+3. package and re-deploy the application
+
+### runtime attach dependency:
+
+Maven:
+```xml
+<dependency>
+  <groupId>co.elastic.otel</groupId>
+  <artifactId>elastic-otel-runtime-attach</artifactId>
+  <version>${VERSION}</version>
+</dependency>
+```
+
+Gradle:
+```
+implementation("co.elastic.otel:elastic-otel-runtime-attach:${VERSION}")
+```
+
+### code modification
+
+A single call to `RuntimeAttach.attachJavaagentToCurrentJvm()` must be added early at the start of the `main` method body.
+Here is an example of a simple spring-boot application:
+
+```java
+@SpringBootApplication
+public class MyApplication {
+
+    public static void main(String[] args) {
+        RuntimeAttach.attachJavaagentToCurrentJvm();
+        SpringApplication.run(MyApplication.class, args);
+    }
+}
+```
+
+## JVM Runtime attach warnings
+
+With recent JVMs, the following warning may be issued in the process standard error output:
+
+```
+WARNING: A Java agent has been loaded dynamically (/tmp/otel-agent6227828786286549290/agent.jar)
+WARNING: If a serviceability tool is in use, please run with -XX:+EnableDynamicAgentLoading to hide this warning
+WARNING: If a serviceability tool is not in use, please run with -Djdk.instrument.traceUsage for more information
+WARNING: Dynamic loading of agents will be disallowed by default in a future release
+```
+
+This message indicates that the dynamic agent attachment will be disabled by in the future, 
+adding `-XX:+EnableDynamicAgentLoading` to the JVM arguments (or `JAVA_TOOL_OPTIONS` env variable) will allow to get rid of it.
+
+It is also safe to ignore it until the runtime attach feature is disabled by default in a newer JVM version and the JVM version used to run the application is updated to it.

runtime-attach/build.gradle.kts

@@ -0,0 +1,28 @@
+plugins {
+  id("elastic-otel.java-conventions")
+  id("elastic-otel.sign-and-publish-conventions")
+}
+
+description = "Elastic Distribution of OpenTelemetry Java Agent - runtime attach"
+
+base.archivesName.set("elastic-otel-runtime-attach")
+
+val agent: Configuration by configurations.creating {
+  isCanBeResolved = true
+  isCanBeConsumed = false
+}
+
+dependencies {
+  implementation(catalog.contribRuntimeAttach)
+  agent(project(":agent"))
+}
+
+tasks {
+  jar {
+    inputs.files(agent)
+    from({
+      agent.singleFile
+    })
+    rename("^(.*)\\.jar\$", "edot-agent.jar")
+  }
+}

runtime-attach/src/main/java/co/elastic/otel/agent/attach/RuntimeAttach.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.otel.agent.attach;
+
+import io.opentelemetry.contrib.attach.core.CoreRuntimeAttach;
+
+/** Provides ability to attach EDOT Java agent to the current JVM at runtime. */
+public class RuntimeAttach {
+
+  /** Attaches EDOT Java agent to the current JVM, must be called early at application startup */
+  public static void attachJavaagentToCurrentJvm() {
+    CoreRuntimeAttach distroRuntimeAttach = new CoreRuntimeAttach("/edot-agent.jar");
+    distroRuntimeAttach.attachJavaagentToCurrentJvm();
+  }
+
+  private RuntimeAttach() {}
+}

settings.gradle.kts

@@ -23,6 +23,7 @@ include("instrumentation:openai-client-instrumentation:instrumentation-0.8")
 include("instrumentation:openai-client-instrumentation:instrumentation-0.14")
 include("inferred-spans")
 include("resources")
+include("runtime-attach")
 include("smoke-tests")
 include("smoke-tests:test-app")
 include("smoke-tests:test-app-war")

smoke-tests/src/test/java/com/example/javaagent/smoketest/DynamicConfigSmokeTest.java

@@ -52,12 +52,12 @@ public static void end() {
   }
 
   @AfterEach
-  public void endTest() throws InterruptedException {
+  public void endTest() {
     doRequest(getUrl("/dynamicconfig/reset"), okResponseBody("reset"));
   }
 
   @Test
-  public void flipSending() throws InterruptedException, IOException {
+  public void flipSending() throws IOException {
     doRequest(getUrl("/health"), okResponseBody("Alive!"));
     doRequest(getUrl("/dynamicconfig/flipSending"), okResponseBody("stopped"));
 

smoke-tests/src/test/java/com/example/javaagent/smoketest/RuntimeAttachSmokeTest.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.example.javaagent.smoketest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import io.opentelemetry.proto.trace.v1.Span;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+public class RuntimeAttachSmokeTest extends TestAppSmokeTest {
+
+  @BeforeAll
+  public static void start() {
+    startTestApp(
+        container -> {
+          String jvmOptions = container.getEnvMap().get("JAVA_TOOL_OPTIONS");
+          if (jvmOptions != null) {
+            // remove '-javaagent' from JVM args
+            jvmOptions =
+                Arrays.asList(jvmOptions.split(" ")).stream()
+                    .map(String::trim)
+                    .filter(s -> !s.isEmpty())
+                    .filter(s -> !s.startsWith("-javaagent:"))
+                    .collect(Collectors.joining());
+            container.withEnv("JAVA_TOOL_OPTIONS", jvmOptions);
+          }
+
+          // make the app use runtime-attach
+          container.withEnv("EDOT_RUNTIME_ATTACH", "true");
+        });
+  }
+
+  @AfterAll
+  public static void end() {
+    stopApp();
+  }
+
+  @Test
+  void runtimeAttachTrace() {
+    // runtime attach is working if we can capture any signal, for simplicity we only test traces
+
+    doRequest(getUrl("/health"), okResponseBody("Alive!"));
+
+    List<ExportTraceServiceRequest> traces = waitForTraces();
+    List<Span> spans = getSpans(traces).toList();
+    assertThat(spans).hasSize(1).extracting("name").containsOnly("GET /health");
+  }
+}

smoke-tests/src/test/java/com/example/javaagent/smoketest/SmokeTest.java

@@ -161,7 +161,7 @@ private static GenericContainer<?> addDockerDebugHost(GenericContainer<?> target
   }
 
   @BeforeEach
-  void beforeEach() throws IOException, InterruptedException {
+  void beforeEach() throws IOException {
     // because traces reporting is asynchronous we need to wait for the healthcheck traces to be
     // reported and only then
     // flush before the test, otherwise the first test will see the healthcheck trace captured.

smoke-tests/test-app/build.gradle.kts

@@ -20,6 +20,8 @@ dependencies {
   implementation("io.opentelemetry:opentelemetry-api")
   implementation("io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations")
 
+  implementation(project(":runtime-attach"))
+
 }
 
 java {