[8.18] [CI] Fix packaging tests by patching system java home (elastic#135504) (elastic#135539)

Older version of openjdk17 are incompatible with newer ubuntu versions.
Packaging tests swap out bundled jdk for the minimum jdk for a few tests.
On ubuntu24 we now replace openjdk17 with adoptopenjdk17 in these cases.

* Fix package upgrade tests for older versions on ubuntu24

* Rework patching JDK in PackageUpgradetests

* Only parse base version

* Fix ubuntu lookup and remove explicit java for upgraded versions

Author: breskeby

.ci/scripts/packaging-test.sh

@@ -69,14 +69,30 @@ sudo mkdir -p /elasticsearch/qa/ && sudo chown jenkins /elasticsearch/qa/ && ln
 # See: https://git-scm.com/docs/git-config/2.35.2#Documentation/git-config.txt-safedirectory
 git config --global --add safe.directory $WORKSPACE
 
+# Older versions of openjdk are incompatible to newer kernel of ubuntu. Use adoptopenjdk17 instead
+resolve_system_java_home() {
+  if [[ "$BUILD_JAVA_HOME" == *"openjdk17"* ]]; then
+    if [ -f "/etc/os-release" ]; then
+        . /etc/os-release
+        if [[ "$ID" == "ubuntu" && "$VERSION_ID" == "24.04" ]]; then
+          echo "$HOME/.java/adoptopenjdk17"
+          return
+        fi
+      fi
+  fi
+
+  echo "$(readlink -f -n $BUILD_JAVA_HOME)"
+}
+
+
 # sudo sets it's own PATH thus we use env to override that and call sudo annother time so we keep the secure root PATH
 # run with --continue to run both bats and java tests even if one fails
 # be explicit about Gradle home dir so we use the same even with sudo
 sudo -E env \
   PATH=$BUILD_JAVA_HOME/bin:`sudo bash -c 'echo -n $PATH'` \
   --unset=ES_JAVA_HOME \
   --unset=JAVA_HOME \
-  SYSTEM_JAVA_HOME=`readlink -f -n $BUILD_JAVA_HOME` \
+  SYSTEM_JAVA_HOME=$(resolve_system_java_home) \
   DOCKER_CONFIG="${HOME}/.docker" \
   ./gradlew -g $HOME/.gradle --console=plain --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@
 

qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java

@@ -87,7 +87,6 @@ public void test12SetupBwcVersion() throws Exception {
         );
 
         assertDocsExist();
-
         stopElasticsearch();
     }
 

qa/packaging/src/test/java/org/elasticsearch/packaging/util/Packages.java

@@ -86,7 +86,7 @@ public static Installation installPackage(Shell sh, Distribution distribution) t
     public static Installation installPackage(Shell sh, Distribution distribution, @Nullable Predicate<String> outputPredicate)
         throws IOException {
         String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout().trim();
-        if (distribution.hasJdk == false) {
+        if (requiresExplicitJavaHome(distribution)) {
             sh.getEnv().put("ES_JAVA_HOME", systemJavaHome);
         }
         final Result result = runPackageManager(distribution, sh, PackageManagerCommand.INSTALL);
@@ -98,7 +98,7 @@ public static Installation installPackage(Shell sh, Distribution distribution, @
         }
         Installation installation = Installation.ofPackage(sh, distribution);
         installation.setElasticPassword(captureElasticPasswordFromOutput(result));
-        if (distribution.hasJdk == false) {
+        if (requiresExplicitJavaHome(distribution)) {
             Files.write(installation.envFile, List.of("ES_JAVA_HOME=" + systemJavaHome), StandardOpenOption.APPEND);
         }
 
@@ -109,6 +109,15 @@ public static Installation installPackage(Shell sh, Distribution distribution, @
         return installation;
     }
 
+    private static boolean requiresExplicitJavaHome(Distribution distribution) {
+        if (distribution.hasJdk == false) {
+            return true;
+        }
+        Version version = Version.fromString(distribution.baseVersion);
+        boolean requiresPatch = Platforms.isUbuntu24() && (version.onOrAfter(Version.V_8_0_0) && version.onOrBefore(Version.V_8_4_3));
+        return requiresPatch;
+    }
+
     private static String captureElasticPasswordFromOutput(Result result) {
         return Arrays.stream(result.stdout().split(System.lineSeparator()))
             .filter(l -> l.contains("The generated password for the elastic built-in superuser is : "))
@@ -123,7 +132,20 @@ public static Installation upgradePackage(Shell sh, Distribution distribution) t
             throw new RuntimeException("Upgrading distribution " + distribution + " failed: " + result);
         }
 
-        return Installation.ofPackage(sh, distribution);
+        Installation installation = Installation.ofPackage(sh, distribution);
+        if (requiresExplicitJavaHome(distribution)) {
+            String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout().trim();
+            Files.write(installation.envFile, List.of("ES_JAVA_HOME=" + systemJavaHome), StandardOpenOption.APPEND);
+        } else {
+            // Explicitly remove the line if added for previous installation
+            Files.write(
+                installation.envFile,
+                Files.readAllLines(installation.envFile).stream().filter(line -> line.startsWith("ES_JAVA_HOME") == false).toList(),
+                StandardOpenOption.WRITE,
+                StandardOpenOption.TRUNCATE_EXISTING
+            );
+        }
+        return installation;
     }
 
     public static Installation forceUpgradePackage(Shell sh, Distribution distribution) throws IOException {

qa/packaging/src/test/java/org/elasticsearch/packaging/util/Platforms.java

@@ -28,6 +28,15 @@ public static String getOsRelease() {
         }
     }
 
+    public static boolean isUbuntu24() {
+        if (LINUX) {
+            String osRelease = getOsRelease();
+            return osRelease.contains("ID=ubuntu") && osRelease.contains("VERSION_ID=\"24.04\"");
+        } else {
+            return false;
+        }
+    }
+
     public static boolean isDPKG() {
         if (WINDOWS) {
             return false;