8339487: ProcessHandleImpl os_getChildren sysctl call - retry in case of ENOMEM and enhance exception message

Backport-of: 4ff72dc57e65e99b129f0ba28196994edf402018

Author: MBaesken

src/java.base/macosx/native/libjava/ProcessHandleImpl_macosx.c

@@ -97,25 +97,35 @@ jint os_getChildren(JNIEnv *env, jlong jpid, jlongArray jarray,
         }
     }
 
-    // Get buffer size needed to read all processes
-    int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_ALL, 0};
-    if (sysctl(mib, 4, NULL, &bufSize, NULL, 0) < 0) {
-        JNU_ThrowByNameWithLastError(env,
-            "java/lang/RuntimeException", "sysctl failed");
-        return -1;
-    }
+    int errsysctl;
+    int maxRetries = 100;
+    void *buffer = NULL;
+    do {
+        int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_ALL, 0};
+        if (buffer != NULL) free(buffer);
+        // Get buffer size needed to read all processes
+        if (sysctl(mib, 4, NULL, &bufSize, NULL, 0) < 0) {
+            JNU_ThrowByNameWithMessageAndLastError(env,
+                "java/lang/RuntimeException", "sysctl failed");
+            return -1;
+        }
 
-    // Allocate buffer big enough for all processes
-    void *buffer = malloc(bufSize);
-    if (buffer == NULL) {
-        JNU_ThrowOutOfMemoryError(env, "malloc failed");
-        return -1;
-    }
+        // Allocate buffer big enough for all processes; add a little
+        // bit of space to be able to hold a few more proc infos
+        // for processes started right after the first sysctl call
+        buffer = malloc(bufSize + 4 * sizeof(struct kinfo_proc));
+        if (buffer == NULL) {
+            JNU_ThrowOutOfMemoryError(env, "malloc failed");
+            return -1;
+        }
 
-    // Read process info for all processes
-    if (sysctl(mib, 4, buffer, &bufSize, NULL, 0) < 0) {
-        JNU_ThrowByNameWithLastError(env,
-            "java/lang/RuntimeException", "sysctl failed");
+        // Read process info for all processes
+        errsysctl = sysctl(mib, 4, buffer, &bufSize, NULL, 0);
+    } while (errsysctl < 0 && errno == ENOMEM && maxRetries-- > 0);
+
+    if (errsysctl < 0) {
+        JNU_ThrowByNameWithMessageAndLastError(env,
+            "java/lang/RuntimeException", "sysctl failed to get info about all processes");
         free(buffer);
         return -1;
     }

src/java.base/unix/native/libjava/ProcessHandleImpl_unix.c

@@ -547,7 +547,7 @@ jint unix_getChildren(JNIEnv *env, jlong jpid, jlongArray jarray,
      * position integer as a filename.
      */
     if ((dir = opendir("/proc")) == NULL) {
-        JNU_ThrowByNameWithLastError(env,
+        JNU_ThrowByNameWithMessageAndLastError(env,
             "java/lang/RuntimeException", "Unable to open /proc");
         return -1;
     }