8305765: CompressedClassPointers.java is unreliable due to ASLR

Reviewed-by: iklam

Author: calvinccheung

src/hotspot/share/memory/metaspace.cpp

@@ -793,6 +793,7 @@ void Metaspace::global_initialize() {
 
     // ...failing that, reserve anywhere, but let platform do optimized placement:
     if (!rs.is_reserved()) {
+      log_info(metaspace)("Reserving compressed class space anywhere");
       rs = Metaspace::reserve_address_space_for_compressed_classes(size, true);
     }
 

test/hotspot/jtreg/ProblemList.txt

@@ -100,7 +100,6 @@ runtime/os/TestTracePageSizes.java#G1 8267460 linux-aarch64
 runtime/os/TestTracePageSizes.java#Parallel 8267460 linux-aarch64
 runtime/os/TestTracePageSizes.java#Serial 8267460 linux-aarch64
 runtime/ErrorHandling/CreateCoredumpOnCrash.java 8267433 macosx-x64
-runtime/CompressedOops/CompressedClassPointers.java 8305765 generic-all
 runtime/StackGuardPages/TestStackGuardPagesNative.java 8303612 linux-all
 runtime/ErrorHandling/TestDwarf.java#checkDecoder 8305489 linux-all
 runtime/ErrorHandling/MachCodeFramesInErrorFile.java 8313315 linux-ppc64le

test/hotspot/jtreg/runtime/CompressedOops/CompressedClassPointers.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -44,7 +44,8 @@
 
 public class CompressedClassPointers {
 
-    static final String logging_option = "-Xlog:gc+metaspace=trace,cds=trace";
+    static final String logging_option = "-Xlog:gc+metaspace=trace,metaspace=info,cds=trace";
+    static final String reserveCCSAnywhere = "Reserving compressed class space anywhere";
 
     // Returns true if we are to test the narrow klass base; we only do this on
     // platforms where we can be reasonably shure that we get reproducable placement).
@@ -56,6 +57,15 @@ static boolean testNarrowKlassBase() {
 
     }
 
+    // Returns true if the output indicates that the ccs is reserved anywhere.
+    static boolean isCCSReservedAnywhere(OutputAnalyzer output) {
+        if (output.getOutput().contains(reserveCCSAnywhere)) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
     // CDS off, small heap, ccs size default (1G)
     // A small heap should allow us to place the ccs within the lower 32G and thus allow zero based encoding.
     public static void smallHeapTest() throws Exception {
@@ -67,7 +77,7 @@ public static void smallHeapTest() throws Exception {
             "-Xshare:off",
             "-XX:+VerifyBeforeGC", "-version");
         OutputAnalyzer output = new OutputAnalyzer(pb.start());
-        if (testNarrowKlassBase()) {
+        if (testNarrowKlassBase() && !isCCSReservedAnywhere(output)) {
             output.shouldContain("Narrow klass base: 0x0000000000000000");
         }
         output.shouldHaveExitValue(0);
@@ -84,7 +94,7 @@ public static void smallHeapTestWith1G() throws Exception {
             "-Xshare:off",
             "-XX:+VerifyBeforeGC", "-version");
         OutputAnalyzer output = new OutputAnalyzer(pb.start());
-        if (testNarrowKlassBase()) {
+        if (testNarrowKlassBase() && !isCCSReservedAnywhere(output)) {
             output.shouldContain("Narrow klass base: 0x0000000000000000, Narrow klass shift: 3");
         }
         output.shouldHaveExitValue(0);
@@ -102,7 +112,7 @@ public static void largeHeapTest() throws Exception {
             "-Xshare:off",
             "-XX:+VerifyBeforeGC", "-version");
         OutputAnalyzer output = new OutputAnalyzer(pb.start());
-        if (testNarrowKlassBase() && !Platform.isPPC() && !Platform.isOSX()) {
+        if (testNarrowKlassBase() && !Platform.isPPC() && !Platform.isOSX() && !isCCSReservedAnywhere(output)) {
             // PPC: in most cases the heap cannot be placed below 32g so there
             // is room for ccs and narrow klass base will be 0x0. Exception:
             // Linux 4.1.42 or earlier (see ELF_ET_DYN_BASE in JDK-8244847).
@@ -128,7 +138,7 @@ public static void largeHeapAbove32GTest() throws Exception {
             "-XX:+VerifyBeforeGC", "-version");
         OutputAnalyzer output = new OutputAnalyzer(pb.start());
         if (testNarrowKlassBase()) {
-            if (!(Platform.isAArch64() && Platform.isOSX())) { // see JDK-8262895
+            if (!(Platform.isAArch64() && Platform.isOSX())  && !isCCSReservedAnywhere(output)) { // see JDK-8262895
                 output.shouldContain("Narrow klass base: 0x0000000000000000");
                 if (!Platform.isAArch64() && !Platform.isPPC() && !Platform.isOSX()) {
                     output.shouldContain("Narrow klass shift: 0");
@@ -211,7 +221,9 @@ public static void smallHeapTestNoCoop() throws Exception {
             "-Xlog:cds=trace",
             "-XX:+VerifyBeforeGC", "-version");
         OutputAnalyzer output = new OutputAnalyzer(pb.start());
-        output.shouldContain("Narrow klass base: 0x0000000000000000");
+        if (!isCCSReservedAnywhere(output)) {
+            output.shouldContain("Narrow klass base: 0x0000000000000000");
+        }
         output.shouldHaveExitValue(0);
     }
 
@@ -227,7 +239,9 @@ public static void smallHeapTestWith1GNoCoop() throws Exception {
             "-Xlog:cds=trace",
             "-XX:+VerifyBeforeGC", "-version");
         OutputAnalyzer output = new OutputAnalyzer(pb.start());
-        output.shouldContain("Narrow klass base: 0x0000000000000000");
+        if (!isCCSReservedAnywhere(output)) {
+            output.shouldContain("Narrow klass base: 0x0000000000000000");
+        }
         if (!Platform.isAArch64() && !Platform.isPPC()) {
             // Currently relax this test for Aarch64 and ppc.
             output.shouldContain("Narrow klass shift: 0");
@@ -247,7 +261,9 @@ public static void largeHeapTestNoCoop() throws Exception {
             "-Xlog:cds=trace",
             "-XX:+VerifyBeforeGC", "-version");
         OutputAnalyzer output = new OutputAnalyzer(pb.start());
-        output.shouldContain("Narrow klass base: 0x0000000000000000");
+        if (!isCCSReservedAnywhere(output)) {
+            output.shouldContain("Narrow klass base: 0x0000000000000000");
+        }
         if (!Platform.isAArch64() && !Platform.isPPC()) {
             // Currently relax this test for Aarch64 and ppc.
             output.shouldContain("Narrow klass shift: 0");