Adding support for latest version to attcher cli (#2659)

Author: eyalkoren

CHANGELOG.asciidoc

@@ -36,6 +36,9 @@ the agent will now include the database name in the dependency, thus `mysql/my-d
 * Added support for the new service target fields - {pull}2578[#2578]
 * Capture the database name from JDBC connection string - {pull}2642[#2642]
 * Added an additional span around Javalin template renderers - {pull}2381[#2381]
+* Added support for downloading the latest agent version through the attach CLI by setting `--download-agent-version latest`. In
+addition, when using the `apm-agent-attach-cli-slim.jar`, which does not contain a bundled agent, the latest version will be downloaded
+from maven at runtime unless configured otherwise through  `--download-agent-version` - {pull}2659[#2659]
 
 [float]
 ===== Bug fixes

apm-agent-attach-cli/src/main/java/co/elastic/apm/attach/AgentAttacher.java

@@ -53,6 +53,7 @@
  */
 public class AgentAttacher {
 
+    public static final String LATEST_VERSION = "latest";
     // intentionally not static so that we can initLogging first
     private final Logger logger = LogManager.getLogger(AgentAttacher.class);
     private final Arguments arguments;
@@ -121,6 +122,10 @@ public static void main(String[] args) {
         Logger logger = initLogging(arguments);
 
         String downloadAgentVersion = arguments.getDownloadAgentVersion();
+        if (downloadAgentVersion == null && arguments.getAgentJar() == null) {
+            // If there is no bundled agent and a path for agent was not specified, default to download the latest
+            downloadAgentVersion = LATEST_VERSION;
+        }
         if (downloadAgentVersion != null) {
             try {
                 downloadAndVerifyAgent(arguments, downloadAgentVersion);
@@ -148,6 +153,9 @@ public static void main(String[] args) {
     }
 
     private static void downloadAndVerifyAgent(Arguments arguments, String downloadAgentVersion) throws Exception {
+        if (downloadAgentVersion.equalsIgnoreCase(LATEST_VERSION)) {
+            downloadAgentVersion = AgentDownloader.findLatestVersion();
+        }
         PgpSignatureVerifier pgpSignatureVerifier;
         try {
             Path targetLibDir = AgentDownloadUtils.of(downloadAgentVersion).getTargetLibDir();
@@ -543,6 +551,7 @@ static void printHelp(PrintStream out) {
             out.println();
             out.println("    --download-agent-version <agent-version>");
             out.println("        Instead of the bundled agent jar, download and attach the specified agent version from maven.");
+            out.println("        <agent-version> can be either the explicit version (for example: `1.15.0`) or `latest`.");
         }
 
         Map<String, String> getConfig() {

apm-agent-attach-cli/src/main/java/co/elastic/apm/attach/AgentDownloader.java

@@ -18,17 +18,23 @@
  */
 package co.elastic.apm.attach;
 
+import co.elastic.apm.agent.util.Version;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Locale;
 import java.util.Properties;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * A utility for downloading any given version of the Elastic APM Java agent from maven central.
@@ -37,6 +43,7 @@
  */
 public class AgentDownloader {
 
+    private static final String VERSION_EXTRACTION_REGEX = "<a href.+>([0-9]+.[0-9.]+.[0-9.]+)/</a>";
     private static final String AGENT_GROUP_ID = "co.elastic.apm";
     private static final String AGENT_ARTIFACT_ID = "elastic-apm-agent";
     private static final String CLI_JAR_VERSION;
@@ -90,7 +97,7 @@ public AgentDownloader(PgpSignatureVerifier pgpSignatureVerifier) {
      */
     Path downloadAndVerifyAgent(String agentVersion) throws Exception {
         logger.debug("Requested to download Elastic APM Java agent version {}", agentVersion);
-        final String mavenAgentBaseUrl = getAgentMavenBaseUrl(agentVersion);
+        final String mavenAgentBaseUrl = getAgentMavenVersionBaseUrl(agentVersion);
         Path targetDir = AgentDownloadUtils.of(agentVersion).getTargetAgentDir();
         String agentJarName = computeAgentJarName(agentVersion);
         String mavenAgentJarUrl = computeFileUrl(mavenAgentBaseUrl, agentJarName);
@@ -133,28 +140,39 @@ String computeAgentJarName(String agentVersion) {
     /**
      * Returns the url for an Elastic APM Agent jar in maven.
      */
-    String getAgentMavenBaseUrl(String agentVersion) throws Exception {
-        final String groupId = AGENT_GROUP_ID.replace(".", "/");
-        final String agentMavenUrl = String.format("https://repo1.maven.org/maven2/%s/%s/%s", groupId, AGENT_ARTIFACT_ID, agentVersion);
+    String getAgentMavenVersionBaseUrl(String agentVersion) throws Exception {
+        final String agentMavenUrl = String.format("%s/%s", getAgentArtifactMavenBaseUrl(), agentVersion);
         if (!verifyUrl(agentMavenUrl)) {
             throw new IllegalArgumentException(String.format("Cannot find maven URL for version %s, make sure provided version is valid", agentVersion));
         }
         return agentMavenUrl;
     }
 
+    /**
+     * Returns the url for the Elastic APM Agent jar artifact in maven.
+     */
+    static String getAgentArtifactMavenBaseUrl() throws Exception {
+        final String groupId = AGENT_GROUP_ID.replace(".", "/");
+        final String agentMavenUrl = String.format("https://repo1.maven.org/maven2/%s/%s", groupId, AGENT_ARTIFACT_ID);
+        if (!verifyUrl(agentMavenUrl)) {
+            throw new IllegalArgumentException(String.format("Cannot find maven URL for agent artifact: %s:%s", groupId, AGENT_ARTIFACT_ID));
+        }
+        return agentMavenUrl;
+    }
+
     /**
      * Returns {@code true} if the given url exists, and {@code false} otherwise.
      * <p>
      * The given url must be {@code https} and existing means a {@code HEAD} request returns 200.
      */
-    private boolean verifyUrl(String urlString) throws IOException {
+    private static boolean verifyUrl(String urlString) throws IOException {
         HttpURLConnection urlConnection = openConnection(urlString);
         urlConnection.setRequestMethod("HEAD");
         urlConnection.connect();
         return urlConnection.getResponseCode() == 200;
     }
 
-    private HttpURLConnection openConnection(String urlString) throws IOException {
+    private static HttpURLConnection openConnection(String urlString) throws IOException {
         URL url = new URL(urlString);
         HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
         urlConnection.addRequestProperty("User-Agent", USER_AGENT);
@@ -201,6 +219,34 @@ void verifyPgpSignature(final Path agentJarFile, final String mavenAgentUrlStrin
         logger.info("Elastic APM Java Agent jar PGP signature successfully verified.");
     }
 
+    static String findLatestVersion() throws Exception {
+        String agentArtifactMavenBaseUrl = getAgentArtifactMavenBaseUrl();
+        HttpURLConnection httpURLConnection = openConnection(agentArtifactMavenBaseUrl);
+        TreeSet<Version> versions = parseMavenArtifactHtml(httpURLConnection.getInputStream());
+        if (versions.isEmpty()) {
+            throw new IllegalStateException("Failed to parse agent versions from the contents of " + agentArtifactMavenBaseUrl);
+        }
+        return versions.last().toString();
+    }
+
+    static TreeSet<Version> parseMavenArtifactHtml(InputStream htmlInputStream) throws IOException {
+        TreeSet<Version> versions = new TreeSet<>();
+        BufferedReader versionsHtmlReader = new BufferedReader(new InputStreamReader(htmlInputStream));
+        Pattern pattern = Pattern.compile(VERSION_EXTRACTION_REGEX);
+        String line;
+        while ((line = versionsHtmlReader.readLine()) != null) {
+            try {
+                Matcher matcher = pattern.matcher(line);
+                if (matcher.find()) {
+                    versions.add(Version.of(matcher.group(1)));
+                }
+            } catch (Exception e) {
+                // ignore, probably a regex false positive
+            }
+        }
+        return versions;
+    }
+
     /**
      * Return the public key ID of our agent signing key.
      *

apm-agent-attach-cli/src/test/java/co/elastic/apm/attach/AgentDownloaderTest.java

@@ -18,13 +18,16 @@
  */
 package co.elastic.apm.attach;
 
+import co.elastic.apm.agent.util.Version;
 import co.elastic.apm.attach.bouncycastle.BouncyCastleVerifier;
 import org.junit.jupiter.api.Test;
 
+import java.io.IOException;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Locale;
+import java.util.TreeSet;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -39,21 +42,21 @@ class AgentDownloaderTest {
 
     @Test
     void testMavenUrl() throws Exception {
-        String mavenUrl = agentDownloader.getAgentMavenBaseUrl("1.24.0");
+        String mavenUrl = agentDownloader.getAgentMavenVersionBaseUrl("1.24.0");
         assertThat(mavenUrl).isNotNull();
         assertThat(mavenUrl).isEqualTo("https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/1.24.0");
     }
 
     @Test
     void testWrongVersion() {
-        assertThatThrownBy(() -> agentDownloader.getAgentMavenBaseUrl("1.24.1")).isInstanceOf(IllegalArgumentException.class);
-        assertThatThrownBy(() -> agentDownloader.getAgentMavenBaseUrl("error")).isInstanceOf(IllegalArgumentException.class);
+        assertThatThrownBy(() -> agentDownloader.getAgentMavenVersionBaseUrl("1.24.1")).isInstanceOf(IllegalArgumentException.class);
+        assertThatThrownBy(() -> agentDownloader.getAgentMavenVersionBaseUrl("error")).isInstanceOf(IllegalArgumentException.class);
     }
 
     @Test
     void testDownloadFile() throws Exception {
         String agentVersion = "1.25.0";
-        String mavenAgentBaseUrl = agentDownloader.getAgentMavenBaseUrl(agentVersion);
+        String mavenAgentBaseUrl = agentDownloader.getAgentMavenVersionBaseUrl(agentVersion);
         String agentPgpSignatureFileName = String.format(Locale.ROOT, "%s.asc", agentDownloader.computeAgentJarName(agentVersion));
         String mavenPgpSignatureUrl = agentDownloader.computeFileUrl(mavenAgentBaseUrl, agentPgpSignatureFileName);
         System.out.println("mavenPgpSignatureUrl = " + mavenPgpSignatureUrl);
@@ -68,7 +71,7 @@ void testDownloadFile() throws Exception {
 
     @Test
     void testDownloadAndVerifyAgent() throws Exception {
-        String agentVersion = "1.25.0";
+        String agentVersion = AgentDownloader.findLatestVersion();
         Path targetDir = AgentDownloadUtils.of(agentVersion).getTargetAgentDir();
         final Path localAgentPath = targetDir.resolve(agentDownloader.computeAgentJarName(agentVersion));
         System.out.println("localAgentPath = " + localAgentPath);
@@ -91,4 +94,20 @@ void testDownloadAgentAndFailVerification() throws Exception {
         assertThatThrownBy(() -> spyAgentDownloader.downloadAndVerifyAgent(agentVersion)).isInstanceOf(IllegalStateException.class);
         assertThat(Files.exists(localAgentPath)).isFalse();
     }
+
+    @Test
+    void testLatestVersion() throws Exception {
+        String latestVersion = AgentDownloader.findLatestVersion();
+        System.out.println("latestVersion = " + latestVersion);
+        assertThat(latestVersion).isNotEmpty();
+    }
+
+    @Test
+    void testAgentArtifactMavenPageParsing() throws IOException {
+        TreeSet<Version> versions = AgentDownloader.parseMavenArtifactHtml(AgentDownloaderTest.class.getResourceAsStream(
+            "/MavenCentral_agent_artifact.html"));
+        assertThat(versions).hasSize(50);
+        assertThat(versions.first().toString()).isEqualTo("0.5.1");
+        assertThat(versions.last().toString()).isEqualTo("1.31.0");
+    }
 }