Refactor identity hashcode access.

Author: christianhaeubl

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java

@@ -48,6 +48,7 @@
 import com.oracle.svm.core.genscavenge.remset.BrickTable;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
 import com.oracle.svm.core.graal.RuntimeCompilation;
+import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.ObjectVisitor;
 import com.oracle.svm.core.log.Log;
@@ -179,7 +180,8 @@ public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHead
             return space.copyAlignedObject(original, originalSpace);
         }
         assert originalSpace == space;
-        Word header = ObjectHeader.readHeaderFromObject(original);
+        ObjectHeader oh = Heap.getHeap().getObjectHeader();
+        Word header = oh.readHeaderFromObject(original);
         if (ObjectHeaderImpl.isMarkedHeader(header)) {
             return original;
         }
@@ -193,7 +195,7 @@ public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHead
              * object's size. The easiest way to handle this is to copy the object.
              */
             result = space.copyAlignedObject(original, originalSpace);
-            assert !ObjectHeaderImpl.hasIdentityHashFromAddressInline(ObjectHeader.readHeaderFromObject(result));
+            assert !ObjectHeaderImpl.hasIdentityHashFromAddressInline(oh.readHeaderFromObject(result));
         }
         ObjectHeaderImpl.setMarked(result);
         markStack.push(result);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java

@@ -31,7 +31,6 @@
 import com.oracle.svm.core.AlwaysInline;
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
-import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.ObjectReferenceVisitor;
 import com.oracle.svm.core.heap.ReferenceAccess;
 import com.oracle.svm.core.hub.LayoutEncoding;
@@ -91,7 +90,7 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole
         // This is the most expensive check as it accesses the heap fairly randomly, which results
         // in a lot of cache misses.
         ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl();
-        Word header = ObjectHeader.readHeaderFromPointer(p);
+        Word header = ohi.readHeaderFromPointer(p);
         if (GCImpl.getGCImpl().isCompleteCollection() || !RememberedSet.get().hasRememberedSet(header)) {
 
             if (ObjectHeaderImpl.isForwardedHeader(header)) {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java

@@ -35,6 +35,7 @@
 import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader;
 import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
+import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.ObjectReferenceVisitor;
 import com.oracle.svm.core.heap.ObjectVisitor;
@@ -241,7 +242,8 @@ private static boolean verifyObject(Object obj, AlignedHeader aChunk, UnalignedH
             return false;
         }
 
-        Word header = ObjectHeader.readHeaderFromPointer(ptr);
+        ObjectHeader oh = Heap.getHeap().getObjectHeader();
+        Word header = oh.readHeaderFromPointer(ptr);
         if (ObjectHeaderImpl.isProducedHeapChunkZapped(header) || ObjectHeaderImpl.isConsumedHeapChunkZapped(header)) {
             Log.log().string("Object ").zhex(ptr).string(" has a zapped header: ").zhex(header).newline();
             return false;
@@ -352,7 +354,8 @@ private static boolean verifyReference(Object parentObject, Pointer reference, P
             return false;
         }
 
-        Word header = ObjectHeader.readHeaderFromPointer(referencedObject);
+        ObjectHeader oh = Heap.getHeap().getObjectHeader();
+        Word header = oh.readHeaderFromPointer(referencedObject);
         if (!ObjectHeaderImpl.getObjectHeaderImpl().isEncodedObjectHeader(header)) {
             Log.log().string("Object reference at ").zhex(reference).string(" does not point to a Java object or the object header of the Java object is invalid: ").zhex(referencedObject)
                             .string(". ");

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java

@@ -156,6 +156,24 @@ protected void initializeObjectHeader(Pointer objectPointer, Word encodedHub, bo
         }
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @Override
+    public Word readHeaderFromPointer(Pointer objectPointer) {
+        if (getReferenceSize() == Integer.BYTES) {
+            return Word.unsigned(objectPointer.readInt(getHubOffset()));
+        }
+        return objectPointer.readWord(getHubOffset());
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @Override
+    public Word readHeaderFromObject(Object o) {
+        if (getReferenceSize() == Integer.BYTES) {
+            return Word.unsigned(ObjectAccess.readInt(o, getHubOffset()));
+        }
+        return ObjectAccess.readWord(o, getHubOffset());
+    }
+
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     @Override
     public boolean hasOptionalIdentityHashField(Word header) {
@@ -337,7 +355,8 @@ public static boolean isAlignedHeader(UnsignedWord header) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isUnalignedObject(Object obj) {
-        UnsignedWord header = readHeaderFromObject(obj);
+        ObjectHeader oh = Heap.getHeap().getObjectHeader();
+        UnsignedWord header = oh.readHeaderFromObject(obj);
         return isUnalignedHeader(header);
     }
 
@@ -348,7 +367,8 @@ public static boolean isUnalignedHeader(UnsignedWord header) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static void setRememberedSetBit(Object o) {
-        UnsignedWord oldHeader = readHeaderFromObject(o);
+        ObjectHeader oh = Heap.getHeap().getObjectHeader();
+        UnsignedWord oldHeader = oh.readHeaderFromObject(o);
         assert oldHeader.and(FORWARDED_OR_MARKED2_BIT).equal(0);
         UnsignedWord newHeader = oldHeader.or(REMSET_OR_MARKED1_BIT);
         writeHeaderToObject(o, newHeader);
@@ -364,7 +384,8 @@ public static void setMarked(Object o) {
         if (!SerialGCOptions.useCompactingOldGen()) { // not guarantee(): always folds, prevent call
             throw VMError.shouldNotReachHere("Only for compacting GC.");
         }
-        UnsignedWord header = readHeaderFromObject(o);
+        ObjectHeader oh = Heap.getHeap().getObjectHeader();
+        UnsignedWord header = oh.readHeaderFromObject(o);
         assert header.and(FORWARDED_OR_MARKED2_BIT).equal(0) : "forwarded or already marked";
         /*
          * The remembered bit is already set if the object was in the old generation before, or
@@ -375,14 +396,16 @@ public static void setMarked(Object o) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static void unsetMarkedAndKeepRememberedSetBit(Object o) {
-        UnsignedWord header = readHeaderFromObject(o);
+        ObjectHeader oh = Heap.getHeap().getObjectHeader();
+        UnsignedWord header = oh.readHeaderFromObject(o);
         assert isMarkedHeader(header);
         writeHeaderToObject(o, header.and(FORWARDED_OR_MARKED2_BIT.not()));
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isMarked(Object o) {
-        return isMarkedHeader(readHeaderFromObject(o));
+        ObjectHeader oh = Heap.getHeap().getObjectHeader();
+        return isMarkedHeader(oh.readHeaderFromObject(o));
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
@@ -395,7 +418,8 @@ public static boolean isMarkedHeader(UnsignedWord header) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     static boolean isPointerToForwardedObject(Pointer p) {
-        Word header = readHeaderFromPointer(p);
+        ObjectHeader oh = Heap.getHeap().getObjectHeader();
+        Word header = oh.readHeaderFromPointer(p);
         return isForwardedHeader(header);
     }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java

@@ -31,7 +31,6 @@
 import java.lang.ref.ReferenceQueue;
 import java.lang.ref.SoftReference;
 
-import jdk.graal.compiler.word.Word;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.UnsignedWord;
 
@@ -47,6 +46,8 @@
 import com.oracle.svm.core.thread.VMOperation;
 import com.oracle.svm.core.util.UnsignedUtils;
 
+import jdk.graal.compiler.word.Word;
+
 /** Discovers and handles {@link Reference} objects during garbage collection. */
 final class ReferenceObjectProcessing {
     /** Head of the linked list of discovered references that need to be revisited. */
@@ -230,7 +231,7 @@ private static boolean processRememberedRef(Reference<?> dr) {
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     private static boolean maybeUpdateForwardedReference(Reference<?> dr, Pointer referentAddr) {
         ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl();
-        UnsignedWord header = ObjectHeader.readHeaderFromPointer(referentAddr);
+        UnsignedWord header = ohi.readHeaderFromPointer(referentAddr);
         if (ObjectHeaderImpl.isForwardedHeader(header)) {
             Object forwardedObj = ohi.getForwardedObject(referentAddr);
             ReferenceInternals.setReferent(dr, forwardedObj);
@@ -261,7 +262,8 @@ static void updateForwardedRefs() {
 
             Pointer refPointer = ReferenceInternals.getReferentPointer(current);
             if (!maybeUpdateForwardedReference(current, refPointer)) {
-                UnsignedWord header = ObjectHeader.readHeaderFromPointer(refPointer);
+                ObjectHeader oh = Heap.getHeap().getObjectHeader();
+                UnsignedWord header = oh.readHeaderFromPointer(refPointer);
                 if (!ObjectHeaderImpl.isMarkedHeader(header)) {
                     ReferenceInternals.setReferent(current, null);
                 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheReachabilityAnalyzer.java

@@ -28,7 +28,6 @@
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.word.Pointer;
 
-import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.ObjectReferenceVisitor;
 import com.oracle.svm.core.heap.ReferenceAccess;
 import com.oracle.svm.core.heap.RuntimeCodeCacheCleaner;
@@ -71,7 +70,7 @@ public static boolean isReachable(Pointer ptrToObj) {
         }
 
         ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl();
-        Word header = ObjectHeader.readHeaderFromPointer(ptrToObj);
+        Word header = ohi.readHeaderFromPointer(ptrToObj);
         if (ObjectHeaderImpl.isForwardedHeader(header)) {
             return true;
         }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java

@@ -40,6 +40,7 @@
 import com.oracle.svm.core.config.ConfigurationValues;
 import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
+import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.ObjectVisitor;
 import com.oracle.svm.core.hub.LayoutEncoding;
@@ -416,7 +417,8 @@ private Object copyAlignedObject(Object originalObj) {
         UnsignedWord copySize = originalSize;
         boolean addIdentityHashField = false;
         if (ConfigurationValues.getObjectLayout().isIdentityHashFieldOptional()) {
-            Word header = ObjectHeader.readHeaderFromObject(originalObj);
+            ObjectHeader oh = Heap.getHeap().getObjectHeader();
+            Word header = oh.readHeaderFromObject(originalObj);
             if (probability(SLOW_PATH_PROBABILITY, ObjectHeaderImpl.hasIdentityHashFromAddressInline(header))) {
                 addIdentityHashField = true;
                 copySize = LayoutEncoding.getSizeFromObjectInlineInGC(originalObj, true);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java

@@ -33,6 +33,7 @@
 import com.oracle.svm.core.AlwaysInline;
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser;
+import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.ObjectVisitor;
 import com.oracle.svm.core.log.Log;
@@ -262,7 +263,8 @@ public boolean contains(Object object) {
             return HeapChunk.getSpace(HeapChunk.getEnclosingHeapChunk(object)).isYoungSpace();
         }
         // Only objects in the young generation have no remembered set
-        UnsignedWord header = ObjectHeader.readHeaderFromObject(object);
+        ObjectHeader oh = Heap.getHeap().getObjectHeader();
+        UnsignedWord header = oh.readHeaderFromObject(object);
         boolean young = !ObjectHeaderImpl.hasRememberedSet(header);
         assert young == HeapChunk.getSpace(HeapChunk.getEnclosingHeapChunk(object)).isYoungSpace();
         return young;

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/PlanningVisitor.java

@@ -35,6 +35,8 @@
 import com.oracle.svm.core.genscavenge.ObjectHeaderImpl;
 import com.oracle.svm.core.genscavenge.Space;
 import com.oracle.svm.core.genscavenge.remset.BrickTable;
+import com.oracle.svm.core.heap.Heap;
+import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.hub.LayoutEncoding;
 
 import jdk.graal.compiler.word.Word;
@@ -77,7 +79,8 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
 
         Pointer p = objSeq;
         while (p.belowThan(initialTop)) {
-            Word header = ObjectHeaderImpl.readHeaderFromPointer(p);
+            ObjectHeader oh = Heap.getHeap().getObjectHeader();
+            Word header = oh.readHeaderFromPointer(p);
 
             UnsignedWord objSize;
             if (ObjectHeaderImpl.isForwardedHeader(header)) {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java

@@ -38,6 +38,7 @@
 import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
 import com.oracle.svm.core.graal.snippets.NodeLoweringProvider;
 import com.oracle.svm.core.graal.snippets.SubstrateTemplates;
+import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.StoredContinuation;
 import com.oracle.svm.core.snippets.SnippetRuntime;
@@ -98,7 +99,8 @@ public static void registerForeignCalls(SubstrateForeignCallsProvider provider)
     @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true)
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static void postWriteBarrierStub(Object object) {
-        UnsignedWord objectHeader = ObjectHeader.readHeaderFromObject(object);
+        ObjectHeader oh = Heap.getHeap().getObjectHeader();
+        UnsignedWord objectHeader = oh.readHeaderFromObject(object);
         if (ObjectHeaderImpl.isUnalignedHeader(objectHeader)) {
             RememberedSet.get().dirtyCardForUnalignedObject(object, false);
         } else {
@@ -112,7 +114,8 @@ public static void postWriteBarrierStub(Object object) {
     @Snippet
     public static void postWriteBarrierSnippet(Object object, @ConstantParameter boolean shouldOutline, @ConstantParameter boolean alwaysAlignedChunk, @ConstantParameter boolean verifyOnly) {
         Object fixedObject = FixedValueAnchorNode.getObject(object);
-        UnsignedWord objectHeader = ObjectHeader.readHeaderFromObject(fixedObject);
+        ObjectHeader oh = Heap.getHeap().getObjectHeader();
+        UnsignedWord objectHeader = oh.readHeaderFromObject(fixedObject);
 
         if (SerialGCOptions.VerifyWriteBarriers.getValue() && alwaysAlignedChunk) {
             /*