diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java index 6c5d6417dd45..8131ddbc8507 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java @@ -24,24 +24,28 @@ */ package com.oracle.svm.core.handles; -import java.lang.ref.WeakReference; - -import jdk.graal.compiler.word.Word; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.genscavenge.GreyToBlackObjRefVisitor; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; +import org.graalvm.nativeimage.ImageInfo; import org.graalvm.nativeimage.ObjectHandle; import org.graalvm.nativeimage.ObjectHandles; +import org.graalvm.nativeimage.c.type.WordPointer; +import org.graalvm.word.LocationIdentity; +import org.graalvm.word.Pointer; import org.graalvm.word.SignedWord; import org.graalvm.word.WordBase; -import jdk.internal.misc.Unsafe; +import com.oracle.svm.core.config.ConfigurationValues; + +import jdk.graal.compiler.word.Word; /** * This class implements {@link ObjectHandle word}-sized integer handles that refer to Java objects. * {@link #create(Object) Creating}, {@link #get(ObjectHandle) dereferencing} and * {@link #destroy(ObjectHandle) destroying} handles is thread-safe and the handles themselves are - * valid across threads. This class also supports weak handles, with which the referenced object may - * be garbage-collected, after which {@link #get(ObjectHandle)} returns {@code null}. Still, weak - * handles must also be {@link #destroyWeak(ObjectHandle) explicitly destroyed} to reclaim their - * handle value. + * valid across threads. *

* The implementation uses a variable number of object arrays, in which each array element * represents a handle. The array element's index determines the handle's integer value, and the @@ -53,13 +57,6 @@ */ public final class ObjectHandlesImpl implements ObjectHandles { - /** Private subclass to distinguish from regular handles to {@link WeakReference} objects. */ - private static final class HandleWeakReference extends WeakReference { - HandleWeakReference(T referent) { - super(referent); - } - } - private static final int MAX_FIRST_BUCKET_CAPACITY = 1024; static { // must be a power of 2 for the arithmetic below to work assert Integer.lowestOneBit(MAX_FIRST_BUCKET_CAPACITY) == MAX_FIRST_BUCKET_CAPACITY; @@ -69,8 +66,10 @@ private static final class HandleWeakReference extends WeakReference { private final SignedWord rangeMax; private final SignedWord nullHandle; - private final Object[][] buckets; + private final WordPointer[] buckets; + private final int[] bucketCapacities; private volatile long unusedHandleSearchIndex = 0; + private int deferredFirstBucketCapacity = -1; public ObjectHandlesImpl() { this(Word.signed(1), Word.signed(Long.MAX_VALUE), Word.signed(0)); @@ -86,12 +85,20 @@ public ObjectHandlesImpl(SignedWord rangeMin, SignedWord rangeMax, SignedWord nu long maxIndex = toIndex(rangeMax); int lastBucketIndex = getBucketIndex(maxIndex); int lastBucketCapacity = getIndexInBucket(maxIndex) + 1; - buckets = new Object[lastBucketIndex + 1][]; + buckets = new WordPointer[lastBucketIndex + 1]; + for (int i = 0; i < buckets.length; i++) { + // Pointer.zero() returns the canonical zero pointer; cast to WordPointer + buckets[i] = Word.zero(); + } + + bucketCapacities = new int[lastBucketIndex + 1]; int firstBucketCapacity = MAX_FIRST_BUCKET_CAPACITY; + if (lastBucketIndex == 0) { // if our range is small, we may have only a single small bucket firstBucketCapacity = lastBucketCapacity; } - buckets[0] = new Object[firstBucketCapacity]; + + deferredFirstBucketCapacity = firstBucketCapacity; } public boolean isInRange(ObjectHandle handle) { @@ -125,15 +132,30 @@ private static int getIndexInBucket(long index) { } private static long getObjectArrayByteOffset(int index) { - return Unsafe.getUnsafe().arrayBaseOffset(Object[].class) + index * Unsafe.getUnsafe().arrayIndexScale(Object[].class); + return (long) index * ConfigurationValues.getTarget().wordSize; + } + + @Uninterruptible(reason = "Called from critical sections") + private static WordPointer allocateBucket(int capacity) { + long bytes = (long) capacity * ConfigurationValues.getTarget().wordSize; + Pointer ptr = NullableNativeMemory.malloc(Word.unsigned(bytes), NmtCategory.JNI); + return (WordPointer) ptr; } - private Object[] getBucket(int bucketIndex) { - // buckets[i] is changed only once from null to its final value: try without volatile first - Object[] bucket = buckets[bucketIndex]; - if (bucket == null) { - bucket = (Object[]) Unsafe.getUnsafe().getReferenceVolatile(buckets, getObjectArrayByteOffset(bucketIndex)); + private WordPointer getBucket(int bucketIndex) { + WordPointer bucket = buckets[bucketIndex]; + if (!bucket.isNull()) { + return bucket; + } + + if (bucketIndex == 0 && deferredFirstBucketCapacity != -1) { + // This is the first time we are accessing the capacity at runtime. + bucketCapacities[0] = deferredFirstBucketCapacity; + deferredFirstBucketCapacity = -1; // Mark as initialized } + + bucket = allocateBucket(bucketCapacities[bucketIndex]); + buckets[bucketIndex] = bucket; return bucket; } @@ -149,6 +171,7 @@ public ObjectHandle create(Object obj) { if (obj == null) { return (ObjectHandle) nullHandle; } + outer: for (;;) { long startIndex = unusedHandleSearchIndex; int startBucketIndex = getBucketIndex(startIndex); @@ -157,12 +180,20 @@ public ObjectHandle create(Object obj) { int bucketIndex = startBucketIndex; int indexInBucket = startIndexInBucket; int lastExistingBucketIndex = -1; - Object[] bucket = getBucket(bucketIndex); + WordPointer bucket = getBucket(bucketIndex); + int bucketCapacity = bucketCapacities[bucketIndex]; + for (;;) { - while (indexInBucket < bucket.length) { - if (bucket[indexInBucket] == null) { - if (Unsafe.getUnsafe().compareAndSetReference(bucket, getObjectArrayByteOffset(indexInBucket), null, obj)) { - int newSearchIndexInBucket = (indexInBucket + 1 < bucket.length) ? (indexInBucket + 1) : indexInBucket; + while (indexInBucket < bucketCapacity) { + long offset = getObjectArrayByteOffset(indexInBucket); + if (bucketCapacities[bucketIndex] == 0) { + throw new IllegalStateException("Bucket not allocated"); + } + + Object currentObj = ((Pointer)bucket).readObject(Word.unsigned(offset)); + if (currentObj == null) { + if (((Pointer)bucket).logicCompareAndSwapObject(Word.unsigned(offset), null, obj, LocationIdentity.ANY_LOCATION)) { + int newSearchIndexInBucket = (indexInBucket + 1 < bucketCapacity) ? (indexInBucket + 1) : indexInBucket; unusedHandleSearchIndex = toIndex(bucketIndex, newSearchIndexInBucket); // (if the next index is in another bucket, we let the next create() // figure it out) @@ -179,7 +210,7 @@ public ObjectHandle create(Object obj) { throw new IllegalStateException("Handle space exhausted"); } int newBucketIndex = lastExistingBucketIndex + 1; - if (getBucket(newBucketIndex) != null) { + if (getBucket(newBucketIndex).isNonNull()) { continue outer; // start over: another thread has created a new bucket } int newBucketCapacity = (MAX_FIRST_BUCKET_CAPACITY << newBucketIndex); @@ -187,20 +218,32 @@ public ObjectHandle create(Object obj) { // last bucket may be smaller newBucketCapacity = getIndexInBucket(maxIndex) + 1; } - Object[] newBucket = new Object[newBucketCapacity]; - Unsafe.getUnsafe().putReferenceVolatile(newBucket, getObjectArrayByteOffset(0), obj); - if (Unsafe.getUnsafe().compareAndSetReference(buckets, getObjectArrayByteOffset(newBucketIndex), null, newBucket)) { + + // Allocate new bucket memory manually + WordPointer newBucket = allocateBucket(newBucketCapacity); + Pointer newBucketPtr = (Pointer) newBucket; + // Initialize the first slot with the object + newBucketPtr.writeObject(Word.unsigned(0), obj); + + // CAS-insert bucket pointer into `buckets[newBucketIndex] + if (newBucketPtr.logicCompareAndSwapObject(Word.unsigned(getObjectArrayByteOffset(newBucketIndex)), null, newBucket, LocationIdentity.ANY_LOCATION)) { + buckets[newBucketIndex] = newBucket; + bucketCapacities[newBucketIndex] = newBucketCapacity; + unusedHandleSearchIndex = toIndex(newBucketIndex, 1); return toHandle(newBucketIndex, 0); + } else { + if (!ImageInfo.inImageCode()) { + NullableNativeMemory.free(newBucketPtr); + } + continue outer; } - // start over: another thread has raced us to create another bucket and won - continue outer; } } bucketIndex++; bucket = getBucket(bucketIndex); - if (bucket == null) { + if (bucket.isNull()) { lastExistingBucketIndex = bucketIndex - 1; bucketIndex = 0; bucket = getBucket(bucketIndex); @@ -210,68 +253,62 @@ public ObjectHandle create(Object obj) { } } - public ObjectHandle createWeak(Object obj) { - return create(new HandleWeakReference<>(obj)); - } - @SuppressWarnings("unchecked") @Override public T get(ObjectHandle handle) { Object obj = doGet(handle); - if (obj instanceof HandleWeakReference) { - obj = ((HandleWeakReference) obj).get(); - } return (T) obj; } private Object doGet(ObjectHandle handle) { - if (handle.equal(nullHandle)) { + if (handle == nullHandle) { return null; } if (!isInRange(handle)) { throw new IllegalArgumentException("Invalid handle"); } long index = toIndex(handle); - Object[] bucket = getBucket(getBucketIndex(index)); - if (bucket == null) { - throw new IllegalArgumentException("Invalid handle"); + WordPointer bucket = getBucket(getBucketIndex(index)); + + if (bucket.isNull()) { + throw new IllegalStateException("Bucket not allocated"); } - int indexInBucket = getIndexInBucket(index); - return Unsafe.getUnsafe().getReferenceVolatile(bucket, getObjectArrayByteOffset(indexInBucket)); - } - public boolean isWeak(ObjectHandle handle) { - return (doGet(handle) instanceof HandleWeakReference); + int indexInBucket = getIndexInBucket(index); + return ((Pointer)bucket).readObject(Word.unsigned(getObjectArrayByteOffset(indexInBucket))); } @Override public void destroy(ObjectHandle handle) { - if (handle.equal(nullHandle)) { + if (handle == nullHandle) { return; } if (!isInRange(handle)) { throw new IllegalArgumentException("Invalid handle"); } long index = toIndex(handle); - Object[] bucket = getBucket(getBucketIndex(index)); - if (bucket == null) { - throw new IllegalArgumentException("Invalid handle"); + WordPointer bucket = getBucket(getBucketIndex(index)); + + if (bucket.isNull()) { + throw new IllegalStateException("Bucket not allocated"); } + int indexInBucket = getIndexInBucket(index); - Unsafe.getUnsafe().putReferenceRelease(bucket, getObjectArrayByteOffset(indexInBucket), null); + ((Pointer)bucket).writeObject(Word.unsigned(getObjectArrayByteOffset(indexInBucket)), null); } - public void destroyWeak(ObjectHandle handle) { - destroy(handle); - } public long computeCurrentCount() { long count = 0; int bucketIndex = 0; - Object[] bucket = getBucket(bucketIndex); - while (bucket != null) { - for (int i = 0; i < bucket.length; i++) { - if (bucket[i] != null) { + long offset = 0; + Object currentObj; + WordPointer bucket = getBucket(bucketIndex); + while (bucket.isNonNull()) { + for (int i = 0; i < bucketCapacities[bucketIndex]; i++) { + offset = getObjectArrayByteOffset(i); + currentObj = ((Pointer)bucket).readObject(Word.unsigned(offset)); + if (currentObj != null) { count++; } } @@ -284,12 +321,28 @@ public long computeCurrentCount() { public long computeCurrentCapacity() { long capacity = 0; int bucketIndex = 0; - Object[] bucket = getBucket(bucketIndex); - while (bucket != null) { - capacity += bucket.length; + WordPointer bucket = getBucket(bucketIndex); + while (bucket.isNonNull()) { + capacity += bucketCapacities[bucketIndex]; bucketIndex++; bucket = getBucket(bucketIndex); } return capacity; } + + public void visitBuckets(GreyToBlackObjRefVisitor visitor) { + for (int i = 0; i < buckets.length; i++) { + Pointer bucket = (Pointer) buckets[i]; + if (bucket.isNonNull()) { + visitor.visitObjectReferences( + bucket, + false, + ConfigurationValues.getTarget().wordSize, + null, + bucketCapacities[i] + ); + } + } + } + } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java index 91b39f6ec6eb..f3f0c0c12ada 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java @@ -243,7 +243,7 @@ public static JNIObjectHandle newGlobalRef(JNIObjectHandle handle) { result = JNIImageHeapHandles.toGlobal(handle); } else { Object obj = getObject(handle); - if (obj != null) { + if (!obj.equals(nullHandle())) { result = JNIGlobalHandles.create(obj); } } @@ -262,7 +262,7 @@ public static JNIObjectHandle newWeakGlobalRef(JNIObjectHandle handle) { result = JNIImageHeapHandles.toWeakGlobal(handle); } else { Object obj = getObject(handle); - if (obj != null) { + if (!obj.equals(nullHandle())) { result = JNIGlobalHandles.createWeak(obj); } } @@ -299,13 +299,29 @@ final class JNIGlobalHandles { assert JNIObjectHandles.nullHandle().equal(Word.zero()); } + // Define the mid-point to split the range in half + private static final SignedWord HANDLE_RANGE_SPLIT_POINT = Word.signed(1L << 30); + private static final int HANDLE_BITS_COUNT = 31; private static final SignedWord HANDLE_BITS_MASK = Word.signed((1L << HANDLE_BITS_COUNT) - 1); private static final int VALIDATION_BITS_SHIFT = HANDLE_BITS_COUNT; - private static final int VALIDATION_BITS_COUNT = 32; + private static final int VALIDATION_BITS_COUNT = 31; private static final SignedWord VALIDATION_BITS_MASK = Word.signed((1L << VALIDATION_BITS_COUNT) - 1).shiftLeft(VALIDATION_BITS_SHIFT); + private static final SignedWord WEAK_HANDLE_FLAG = Word.signed(1L << 62); private static final SignedWord MSB = Word.signed(1L << 63); - private static final ObjectHandlesImpl globalHandles = new ObjectHandlesImpl(JNIObjectHandles.nullHandle().add(1), HANDLE_BITS_MASK, JNIObjectHandles.nullHandle()); + + // Strong global handles will occupy the lower half of the global handles range + public static final SignedWord STRONG_GLOBAL_RANGE_MIN = JNIObjectHandles.nullHandle().add(1);; + public static final SignedWord STRONG_GLOBAL_RANGE_MAX = HANDLE_RANGE_SPLIT_POINT.subtract(1); + + // Weak global handles will occupy the upper half of the global handles range + public static final SignedWord WEAK_GLOBAL_RANGE_MIN = HANDLE_RANGE_SPLIT_POINT; + public static final SignedWord WEAK_GLOBAL_RANGE_MAX = HANDLE_BITS_MASK; + + private static final ObjectHandlesImpl strongGlobalHandles + = new ObjectHandlesImpl(STRONG_GLOBAL_RANGE_MIN, STRONG_GLOBAL_RANGE_MAX, JNIObjectHandles.nullHandle()); + private static final ObjectHandlesImpl weakGlobalHandles + = new ObjectHandlesImpl(WEAK_GLOBAL_RANGE_MIN, WEAK_GLOBAL_RANGE_MAX, JNIObjectHandles.nullHandle()); @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static boolean isInRange(JNIObjectHandle handle) { @@ -317,7 +333,20 @@ private static Word isolateHash() { return Word.unsigned(isolateHash); } - private static JNIObjectHandle encode(ObjectHandle handle) { + /** + * Encodes a raw {@code ObjectHandle} into a strong {@code JNIObjectHandle}. + * * A strong handle guarantees the referenced object remains alive as long as + * the handle itself exists. + * * The handle is encoded by: + * 1. Asserting the handle fits within the available bit range. + * 2. Inserting validation bits (derived from the isolate hash) for security. + * 3. Setting the Most Significant Bit (MSB, bit 63) to mark it as an encoded handle. + * 4. The WEAK_HANDLE_FLAG bit (bit 62) remains 0. + * + * @param handle The raw, unencoded handle to the Java object. + * @return The resulting strong JNI object handle with embedded metadata. + */ + private static JNIObjectHandle encodeStrong(ObjectHandle handle) { SignedWord h = (Word) handle; if (JNIObjectHandles.haveAssertions()) { assert h.and(HANDLE_BITS_MASK).equal(h) : "unencoded handle must fit in range"; @@ -330,6 +359,24 @@ private static JNIObjectHandle encode(ObjectHandle handle) { return (JNIObjectHandle) h; } + /** + * Encodes a raw {@code ObjectHandle} into a weak {@code JNIObjectHandle}. + * * A weak handle allows the referenced object to be garbage collected even + * if the handle exists. The handle will be cleared when the object dies. + * * This method calls {@link #encodeStrong(ObjectHandle)} to perform all + * common encoding steps, and then explicitly sets the {@code WEAK_HANDLE_FLAG} + * bit (bit 62) to mark the handle as weak. + * + * @param handle The raw, unencoded handle to the Java object. + * @return The resulting weak JNI object handle with embedded metadata. + */ + private static JNIObjectHandle encodeWeak(ObjectHandle handle) { + SignedWord h = (Word) encodeStrong(handle); + h = h.or(WEAK_HANDLE_FLAG); + assert isInRange((JNIObjectHandle) h); + return (JNIObjectHandle) h; + } + private static ObjectHandle decode(JNIObjectHandle handle) { assert isInRange(handle); assert ((Word) handle).and(VALIDATION_BITS_MASK).unsignedShiftRight(VALIDATION_BITS_SHIFT) @@ -338,35 +385,52 @@ private static ObjectHandle decode(JNIObjectHandle handle) { } static T getObject(JNIObjectHandle handle) { - return globalHandles.get(decode(handle)); + SignedWord handleValue = (Word) handle; + if (handleValue.greaterOrEqual(STRONG_GLOBAL_RANGE_MIN) && + handleValue.lessOrEqual(STRONG_GLOBAL_RANGE_MAX)) { + return strongGlobalHandles.get(decode(handle)); + } + + if (handleValue.greaterOrEqual(WEAK_GLOBAL_RANGE_MIN) && + handleValue.lessOrEqual(WEAK_GLOBAL_RANGE_MAX)) { + return weakGlobalHandles.get(decode((handle))); + } + + throw new IllegalArgumentException("Invalid handle"); } static JNIObjectRefType getHandleType(JNIObjectHandle handle) { - assert isInRange(handle); - if (globalHandles.isWeak(decode(handle))) { + SignedWord handleValue = (Word) handle; + if (handleValue.greaterOrEqual(STRONG_GLOBAL_RANGE_MIN) && + handleValue.lessOrEqual(STRONG_GLOBAL_RANGE_MAX)) { + return JNIObjectRefType.Global; + } + + if (handleValue.greaterOrEqual(WEAK_GLOBAL_RANGE_MIN) && + handleValue.lessOrEqual(WEAK_GLOBAL_RANGE_MAX)) { return JNIObjectRefType.WeakGlobal; } - return JNIObjectRefType.Global; + return JNIObjectRefType.Invalid; } static JNIObjectHandle create(Object obj) { - return encode(globalHandles.create(obj)); + return encodeStrong(strongGlobalHandles.create(obj)); } static void destroy(JNIObjectHandle handle) { - globalHandles.destroy(decode(handle)); + strongGlobalHandles.destroy(decode(handle)); } static JNIObjectHandle createWeak(Object obj) { - return encode(globalHandles.createWeak(obj)); + return encodeWeak(weakGlobalHandles.create(obj)); } static void destroyWeak(JNIObjectHandle weakRef) { - globalHandles.destroyWeak(decode(weakRef)); + weakGlobalHandles.destroy(decode(weakRef)); } public static long computeCurrentCount() { - return globalHandles.computeCurrentCount(); + return strongGlobalHandles.computeCurrentCount() + weakGlobalHandles.computeCurrentCount(); } }