8294316: SA core file support is broken on macosx-x64 starting with macOS 12.x

Reviewed-by: sgehwolf
Backport-of: eaa475f72ced51cf0f0d0bce9ff1265bfb81ab31

Author: vieiro

src/jdk.hotspot.agent/macosx/native/libsaproc/ps_core.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -622,11 +622,16 @@ static bool read_core_segments(struct ps_prochandle* ph) {
         print_debug("failed to read LC_SEGMENT_64 i = %d!\n", i);
         goto err;
       }
-      if (add_map_info(ph, fd, segcmd.fileoff, segcmd.vmaddr, segcmd.vmsize) == NULL) {
-        print_debug("Failed to add map_info at i = %d\n", i);
-        goto err;
+      // The base of the library is offset by a random amount which ends up as a load command with a
+      // filesize of 0.  This must be ignored otherwise the base address of the library is wrong.
+      if (segcmd.filesize != 0) {
+        if (add_map_info(ph, fd, segcmd.fileoff, segcmd.vmaddr, segcmd.vmsize) == NULL) {
+          print_debug("Failed to add map_info at i = %d\n", i);
+          goto err;
+        }
       }
-      print_debug("LC_SEGMENT_64 added: nsects=%d fileoff=0x%llx vmaddr=0x%llx vmsize=0x%llx filesize=0x%llx %s\n",
+      print_debug("LC_SEGMENT_64 %s: nsects=%d fileoff=0x%llx vmaddr=0x%llx vmsize=0x%llx filesize=0x%llx %s\n",
+                  segcmd.filesize == 0 ? "with filesize == 0 ignored" : "added",
                   segcmd.nsects, segcmd.fileoff, segcmd.vmaddr, segcmd.vmsize,
                   segcmd.filesize, &segcmd.segname[0]);
     } else if (lcmd.cmd == LC_THREAD || lcmd.cmd == LC_UNIXTHREAD) {

test/hotspot/jtreg/ProblemList.txt

@@ -164,22 +164,22 @@ runtime/jni/terminatedThread/TestTerminatedThread.java 8219652 aix-ppc64
 
 serviceability/dcmd/gc/RunFinalizationTest.java 8227120 generic-all
 serviceability/sa/ClhsdbAttach.java 8193639 solaris-all
-serviceability/sa/ClhsdbCDSCore.java 8294316,8193639,8267433 solaris-all,macosx-x64
+serviceability/sa/ClhsdbCDSCore.java 8193639,8267433 solaris-all,macosx-x64
 serviceability/sa/ClhsdbCDSJstackPrintAll.java 8193639 solaris-all
 serviceability/sa/CDSJMapClstats.java 8193639 solaris-all
 serviceability/sa/ClhsdbField.java 8193639 solaris-all
-serviceability/sa/ClhsdbFindPC.java 8294316,8193639,8267433 solaris-all,macosx-x64
+serviceability/sa/ClhsdbFindPC.java 8193639,8267433 solaris-all,macosx-x64
 serviceability/sa/ClhsdbFlags.java 8193639 solaris-all
 serviceability/sa/ClhsdbInspect.java 8193639 solaris-all
 serviceability/sa/ClhsdbJdis.java 8193639 solaris-all
 serviceability/sa/ClhsdbJhisto.java 8193639 solaris-all
 serviceability/sa/ClhsdbJstack.java 8193639 solaris-all
 serviceability/sa/ClhsdbLongConstant.java 8193639 solaris-all
-serviceability/sa/ClhsdbPmap.java 8294316,8193639,8267433 solaris-all,macosx-x64
+serviceability/sa/ClhsdbPmap.java 8193639,8267433 solaris-all,macosx-x64
 serviceability/sa/ClhsdbPrintAll.java 8193639 solaris-all
 serviceability/sa/ClhsdbPrintAs.java 8193639 solaris-all
 serviceability/sa/ClhsdbPrintStatics.java 8193639 solaris-all
-serviceability/sa/ClhsdbPstack.java 8294316,8193639,8267433 solaris-all,macosx-x64
+serviceability/sa/ClhsdbPstack.java 8193639,8267433 solaris-all,macosx-x64
 serviceability/sa/ClhsdbRegionDetailsScanOopsForG1.java 8193639 solaris-all
 serviceability/sa/ClhsdbScanOops.java 8193639 solaris-all
 serviceability/sa/ClhsdbSource.java 8193639 solaris-all
@@ -203,8 +203,8 @@ serviceability/sa/TestInstanceKlassSize.java 8193639 solaris-all
 serviceability/sa/TestInstanceKlassSizeForInterface.java 8193639 solaris-all
 serviceability/sa/TestIntConstant.java 8193639 solaris-all
 serviceability/sa/TestJhsdbJstackLock.java 8193639 solaris-all
-serviceability/sa/TestJmapCore.java 8294316,8193639,8267433 solaris-all,macosx-x64
-serviceability/sa/TestJmapCoreMetaspace.java 8294316,8193639,8267433 solaris-all,macosx-x64
+serviceability/sa/TestJmapCore.java 8193639,8267433 solaris-all,macosx-x64
+serviceability/sa/TestJmapCoreMetaspace.java 8193639,8267433 solaris-all,macosx-x64
 serviceability/sa/TestPrintMdo.java 8193639 solaris-all
 serviceability/sa/TestRevPtrsForInvokeDynamic.java 8191270 generic-all
 serviceability/sa/TestType.java 8193639 solaris-all