Look for -XX:[+/-]UseZlibNX on AIX

AIX puts zlibNX on the LIBPATH when running on P9 or later, but having
this in the environment may cause some things, such as git clone, to
fail when exec'ed from Java. For example,
`fatal: pack has bad object at offset 3872100: inflate returned -5`

Backport of ibmruntimes/openj9-openjdk-jdk#664

Signed-off-by: Peter Shipton <[email protected]>

Author: pshipton

src/java.base/unix/native/libjli/java_md.c

@@ -25,7 +25,7 @@
 
 /*
  * ===========================================================================
- * (c) Copyright IBM Corp. 2020, 2021 All Rights Reserved
+ * (c) Copyright IBM Corp. 2020, 2023 All Rights Reserved
  * ===========================================================================
  */
 
@@ -376,12 +376,35 @@ CreateExecutionEnvironment(int *pargc, char ***pargv,
          *     o          $JVMPATH (directory portion only)
          *     o          $JRE/lib
          *     o          $JRE/../lib
-         *     o          ZLIBNX_PATH (for AIX P9 or newer systems with NX)
+         *     o          ZLIBNX_PATH (for AIX P9 or newer systems with NX, unless -XX:-UseZlibNX is set)
          *
          * followed by the user's previous effective LD_LIBRARY_PATH, if
          * any.
          */
 
+#ifdef AIX
+        int aixargc = *pargc - 1; // skip the launcher name
+        char **aixargv = *pargv + 1;
+        const char *aixarg = NULL;
+        jboolean useZlibNX = JNI_TRUE;
+        while (aixargc > 0 && *(aixarg = *aixargv) == '-') {
+            if (JLI_StrCmp(aixarg, "-XX:+UseZlibNX") == 0) {
+                useZlibNX = JNI_TRUE;
+            } else if (JLI_StrCmp(aixarg, "-XX:-UseZlibNX") == 0) {
+                useZlibNX = JNI_FALSE;
+            }
+            aixargc--;
+            aixargv++;
+        }
+        useZlibNX = useZlibNX && power_9_andup() && power_nx_gzip();
+        if (JLI_IsTraceLauncher()) {
+            printf("Add " ZLIBNX_PATH " to the LIBPATH: %s P9+ %s NX %s\n",
+                useZlibNX ? "TRUE" : "FALSE",
+                power_9_andup() ? "TRUE" : "FALSE",
+                power_nx_gzip() ? "TRUE" : "FALSE");
+        }
+#endif
+
         runpath = getenv(LD_LIBRARY_PATH);
 
         /* runpath contains current effective LD_LIBRARY_PATH setting */
@@ -390,8 +413,8 @@ CreateExecutionEnvironment(int *pargc, char ***pargv,
             new_runpath_size = ((runpath != NULL) ? JLI_StrLen(runpath) : 0) +
                     2 * JLI_StrLen(jrepath) +
 #ifdef AIX
-                    /* On AIX P9 or newer with NX accelerator enabled, add the accelerated zlibNX to LIBPATH */
-                    ((power_9_andup() && power_nx_gzip()) ? JLI_StrLen(":" ZLIBNX_PATH) : 0) +
+                    /* On AIX P9 or newer with NX accelerator enabled, add the accelerated zlibNX to LIBPATH. */
+                    (useZlibNX ? JLI_StrLen(":" ZLIBNX_PATH) : 0) +
 #endif
                     JLI_StrLen(new_jvmpath) + 52;
             new_runpath = JLI_MemAlloc(new_runpath_size);
@@ -419,7 +442,7 @@ CreateExecutionEnvironment(int *pargc, char ***pargv,
                         jrepath,
                         jrepath
 #ifdef AIX
-                        , ((power_9_andup() && power_nx_gzip()) ? (":" ZLIBNX_PATH) : "")
+                        , (useZlibNX ? (":" ZLIBNX_PATH) : "")
 #endif
                         );