From d200a655347dfbe4ffca94f84201a06e7ed14133 Mon Sep 17 00:00:00 2001 From: asselyam Date: Wed, 29 Oct 2025 23:47:30 +0100 Subject: [PATCH 01/10] Implement native object handles feature --- .../svm/core/handles/ObjectHandlesImpl.java | 26 +----- .../oracle/svm/core/jni/JNIObjectHandles.java | 83 ++++++++++++++++--- 2 files changed, 72 insertions(+), 37 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java index 6c5d6417dd45..6f9927636773 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java @@ -38,10 +38,7 @@ * This class implements {@link ObjectHandle word}-sized integer handles that refer to Java objects. * {@link #create(Object) Creating}, {@link #get(ObjectHandle) dereferencing} and * {@link #destroy(ObjectHandle) destroying} handles is thread-safe and the handles themselves are - * valid across threads. This class also supports weak handles, with which the referenced object may - * be garbage-collected, after which {@link #get(ObjectHandle)} returns {@code null}. Still, weak - * handles must also be {@link #destroyWeak(ObjectHandle) explicitly destroyed} to reclaim their - * handle value. + * valid across threads. *

* The implementation uses a variable number of object arrays, in which each array element * represents a handle. The array element's index determines the handle's integer value, and the @@ -53,13 +50,6 @@ */ public final class ObjectHandlesImpl implements ObjectHandles { - /** Private subclass to distinguish from regular handles to {@link WeakReference} objects. */ - private static final class HandleWeakReference extends WeakReference { - HandleWeakReference(T referent) { - super(referent); - } - } - private static final int MAX_FIRST_BUCKET_CAPACITY = 1024; static { // must be a power of 2 for the arithmetic below to work assert Integer.lowestOneBit(MAX_FIRST_BUCKET_CAPACITY) == MAX_FIRST_BUCKET_CAPACITY; @@ -210,17 +200,10 @@ public ObjectHandle create(Object obj) { } } - public ObjectHandle createWeak(Object obj) { - return create(new HandleWeakReference<>(obj)); - } - @SuppressWarnings("unchecked") @Override public T get(ObjectHandle handle) { Object obj = doGet(handle); - if (obj instanceof HandleWeakReference) { - obj = ((HandleWeakReference) obj).get(); - } return (T) obj; } @@ -240,10 +223,6 @@ private Object doGet(ObjectHandle handle) { return Unsafe.getUnsafe().getReferenceVolatile(bucket, getObjectArrayByteOffset(indexInBucket)); } - public boolean isWeak(ObjectHandle handle) { - return (doGet(handle) instanceof HandleWeakReference); - } - @Override public void destroy(ObjectHandle handle) { if (handle.equal(nullHandle)) { @@ -261,9 +240,6 @@ public void destroy(ObjectHandle handle) { Unsafe.getUnsafe().putReferenceRelease(bucket, getObjectArrayByteOffset(indexInBucket), null); } - public void destroyWeak(ObjectHandle handle) { - destroy(handle); - } public long computeCurrentCount() { long count = 0; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java index 91b39f6ec6eb..92910e55f4b0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java @@ -34,7 +34,6 @@ import com.oracle.svm.core.RuntimeAssertionsSupport; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.handles.ObjectHandlesImpl; import com.oracle.svm.core.handles.ThreadLocalHandles; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.jni.headers.JNIObjectHandle; @@ -299,13 +298,29 @@ final class JNIGlobalHandles { assert JNIObjectHandles.nullHandle().equal(Word.zero()); } + // Define the mid-point to split the range in half + private static final SignedWord HANDLE_RANGE_SPLIT_POINT = Word.signed(1L << 30); + private static final int HANDLE_BITS_COUNT = 31; private static final SignedWord HANDLE_BITS_MASK = Word.signed((1L << HANDLE_BITS_COUNT) - 1); private static final int VALIDATION_BITS_SHIFT = HANDLE_BITS_COUNT; - private static final int VALIDATION_BITS_COUNT = 32; + private static final int VALIDATION_BITS_COUNT = 31; private static final SignedWord VALIDATION_BITS_MASK = Word.signed((1L << VALIDATION_BITS_COUNT) - 1).shiftLeft(VALIDATION_BITS_SHIFT); + private static final SignedWord WEAK_HANDLE_FLAG = Word.signed(1L << 62); private static final SignedWord MSB = Word.signed(1L << 63); - private static final ObjectHandlesImpl globalHandles = new ObjectHandlesImpl(JNIObjectHandles.nullHandle().add(1), HANDLE_BITS_MASK, JNIObjectHandles.nullHandle()); + + // Strong global handles will occupy the lower half of the global handles range + public static final SignedWord STRONG_GLOBAL_RANGE_MIN = JNIObjectHandles.nullHandle().add(1);; + public static final SignedWord STRONG_GLOBAL_RANGE_MAX = HANDLE_RANGE_SPLIT_POINT.subtract(1); + + // Weak global handles will occupy the upper half of the global handles range + public static final SignedWord WEAK_GLOBAL_RANGE_MIN = HANDLE_RANGE_SPLIT_POINT; + public static final SignedWord WEAK_GLOBAL_RANGE_MAX = HANDLE_BITS_MASK; + + private static final ObjectHandlesImpl strongGlobalHandles + = new ObjectHandlesImpl(STRONG_GLOBAL_RANGE_MIN, STRONG_GLOBAL_RANGE_MAX, JNIObjectHandles.nullHandle()); + private static final ObjectHandlesImpl weakGlobalHandles + = new ObjectHandlesImpl(WEAK_GLOBAL_RANGE_MIN, WEAK_GLOBAL_RANGE_MAX, JNIObjectHandles.nullHandle()); @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static boolean isInRange(JNIObjectHandle handle) { @@ -317,6 +332,19 @@ private static Word isolateHash() { return Word.unsigned(isolateHash); } + /** + * Encodes a raw {@code ObjectHandle} into a strong {@code JNIObjectHandle}. + * * A strong handle guarantees the referenced object remains alive as long as + * the handle itself exists. + * * The handle is encoded by: + * 1. Asserting the handle fits within the available bit range. + * 2. Inserting validation bits (derived from the isolate hash) for security. + * 3. Setting the Most Significant Bit (MSB, bit 63) to mark it as an encoded handle. + * 4. The WEAK_HANDLE_FLAG bit (bit 62) remains 0. + * + * @param handle The raw, unencoded handle to the Java object. + * @return The resulting strong JNI object handle with embedded metadata. + */ private static JNIObjectHandle encode(ObjectHandle handle) { SignedWord h = (Word) handle; if (JNIObjectHandles.haveAssertions()) { @@ -330,6 +358,24 @@ private static JNIObjectHandle encode(ObjectHandle handle) { return (JNIObjectHandle) h; } + /** + * Encodes a raw {@code ObjectHandle} into a weak {@code JNIObjectHandle}. + * * A weak handle allows the referenced object to be garbage collected even + * if the handle exists. The handle will be cleared when the object dies. + * * This method calls {@link #encodeStrong(ObjectHandle)} to perform all + * common encoding steps, and then explicitly sets the {@code WEAK_HANDLE_FLAG} + * bit (bit 62) to mark the handle as weak. + * + * @param handle The raw, unencoded handle to the Java object. + * @return The resulting weak JNI object handle with embedded metadata. + */ + private static JNIObjectHandle encodeWeak(ObjectHandle handle) { + SignedWord h = (Word) encodeStrong(handle); + h = h.or(WEAK_HANDLE_FLAG); + assert isInRange((JNIObjectHandle) h); + return (JNIObjectHandle) h; + } + private static ObjectHandle decode(JNIObjectHandle handle) { assert isInRange(handle); assert ((Word) handle).and(VALIDATION_BITS_MASK).unsignedShiftRight(VALIDATION_BITS_SHIFT) @@ -338,35 +384,48 @@ private static ObjectHandle decode(JNIObjectHandle handle) { } static T getObject(JNIObjectHandle handle) { - return globalHandles.get(decode(handle)); + SignedWord handleValue = (Word) handle; + if ((handleValue.toLong() & WEAK_HANDLE_FLAG.toLong()) == 0) { + return strongGlobalHandles.get(decode(handle)); + } + + if ((handleValue.toLong() & WEAK_HANDLE_FLAG.toLong()) == 1) { + return weakGlobalHandles.get(decode((handle))); + } + + throw new IllegalArgumentException("Invalid handle"); } static JNIObjectRefType getHandleType(JNIObjectHandle handle) { - assert isInRange(handle); - if (globalHandles.isWeak(decode(handle))) { + SignedWord handleValue = (Word) handle; + if ((handleValue.toLong() & WEAK_HANDLE_FLAG.toLong()) == 0) { + return JNIObjectRefType.Global; + } + + if ((handleValue.toLong() & WEAK_HANDLE_FLAG.toLong()) == 1) { return JNIObjectRefType.WeakGlobal; } - return JNIObjectRefType.Global; + return JNIObjectRefType.Invalid; } static JNIObjectHandle create(Object obj) { - return encode(globalHandles.create(obj)); + return encode(strongGlobalHandles.create(obj)); } static void destroy(JNIObjectHandle handle) { - globalHandles.destroy(decode(handle)); + strongGlobalHandles.destroy(decode(handle)); } static JNIObjectHandle createWeak(Object obj) { - return encode(globalHandles.createWeak(obj)); + return encodeWeak(weakGlobalHandles.create(obj)); } static void destroyWeak(JNIObjectHandle weakRef) { - globalHandles.destroyWeak(decode(weakRef)); + weakGlobalHandles.destroy(decode(weakRef)); } public static long computeCurrentCount() { - return globalHandles.computeCurrentCount(); + return strongGlobalHandles.computeCurrentCount() + weakGlobalHandles.computeCurrentCount(); } } From ca5edbea021eabbbf3c14ecc77ec6789ebddba92 Mon Sep 17 00:00:00 2001 From: asselyam Date: Thu, 13 Nov 2025 10:17:19 +0100 Subject: [PATCH 02/10] Use of native-memory instead of java heap --- .../svm/core/handles/ObjectHandlesImpl.java | 101 +++++++++++------- .../oracle/svm/core/jni/JNIObjectHandles.java | 21 ++-- 2 files changed, 75 insertions(+), 47 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java index 6f9927636773..ce9989f34f5c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java @@ -24,15 +24,19 @@ */ package com.oracle.svm.core.handles; -import java.lang.ref.WeakReference; - -import jdk.graal.compiler.word.Word; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import org.graalvm.nativeimage.ObjectHandle; import org.graalvm.nativeimage.ObjectHandles; +import org.graalvm.nativeimage.c.type.WordPointer; +import org.graalvm.word.LocationIdentity; +import org.graalvm.word.Pointer; import org.graalvm.word.SignedWord; import org.graalvm.word.WordBase; -import jdk.internal.misc.Unsafe; +import com.oracle.svm.core.config.ConfigurationValues; + +import jdk.graal.compiler.word.Word; /** * This class implements {@link ObjectHandle word}-sized integer handles that refer to Java objects. @@ -51,6 +55,7 @@ public final class ObjectHandlesImpl implements ObjectHandles { private static final int MAX_FIRST_BUCKET_CAPACITY = 1024; + private static final int wordSize = ConfigurationValues.getTarget().wordSize; static { // must be a power of 2 for the arithmetic below to work assert Integer.lowestOneBit(MAX_FIRST_BUCKET_CAPACITY) == MAX_FIRST_BUCKET_CAPACITY; } @@ -59,7 +64,8 @@ public final class ObjectHandlesImpl implements ObjectHandles { private final SignedWord rangeMax; private final SignedWord nullHandle; - private final Object[][] buckets; + private final WordPointer[] buckets; + private final int[] bucketCapacities; private volatile long unusedHandleSearchIndex = 0; public ObjectHandlesImpl() { @@ -76,12 +82,17 @@ public ObjectHandlesImpl(SignedWord rangeMin, SignedWord rangeMax, SignedWord nu long maxIndex = toIndex(rangeMax); int lastBucketIndex = getBucketIndex(maxIndex); int lastBucketCapacity = getIndexInBucket(maxIndex) + 1; - buckets = new Object[lastBucketIndex + 1][]; + buckets = new WordPointer[lastBucketIndex + 1]; + bucketCapacities = new int[lastBucketIndex + 1]; int firstBucketCapacity = MAX_FIRST_BUCKET_CAPACITY; + if (lastBucketIndex == 0) { // if our range is small, we may have only a single small bucket firstBucketCapacity = lastBucketCapacity; } - buckets[0] = new Object[firstBucketCapacity]; + + Pointer bucketBasePtr = NullableNativeMemory.malloc(Word.unsigned(firstBucketCapacity * wordSize), NmtCategory.JNI); + buckets[0] = (WordPointer) bucketBasePtr; + bucketCapacities[0] = firstBucketCapacity; } public boolean isInRange(ObjectHandle handle) { @@ -115,15 +126,13 @@ private static int getIndexInBucket(long index) { } private static long getObjectArrayByteOffset(int index) { - return Unsafe.getUnsafe().arrayBaseOffset(Object[].class) + index * Unsafe.getUnsafe().arrayIndexScale(Object[].class); + return (long) index * wordSize; } - private Object[] getBucket(int bucketIndex) { + private WordPointer getBucket(int bucketIndex) { // buckets[i] is changed only once from null to its final value: try without volatile first - Object[] bucket = buckets[bucketIndex]; - if (bucket == null) { - bucket = (Object[]) Unsafe.getUnsafe().getReferenceVolatile(buckets, getObjectArrayByteOffset(bucketIndex)); - } + WordPointer bucket = buckets[bucketIndex]; + return bucket; } @@ -139,6 +148,7 @@ public ObjectHandle create(Object obj) { if (obj == null) { return (ObjectHandle) nullHandle; } + outer: for (;;) { long startIndex = unusedHandleSearchIndex; int startBucketIndex = getBucketIndex(startIndex); @@ -147,12 +157,16 @@ public ObjectHandle create(Object obj) { int bucketIndex = startBucketIndex; int indexInBucket = startIndexInBucket; int lastExistingBucketIndex = -1; - Object[] bucket = getBucket(bucketIndex); + Pointer bucket = (Pointer) getBucket(bucketIndex); + int bucketCapacity = bucketCapacities[bucketIndex]; + for (;;) { - while (indexInBucket < bucket.length) { - if (bucket[indexInBucket] == null) { - if (Unsafe.getUnsafe().compareAndSetReference(bucket, getObjectArrayByteOffset(indexInBucket), null, obj)) { - int newSearchIndexInBucket = (indexInBucket + 1 < bucket.length) ? (indexInBucket + 1) : indexInBucket; + while (indexInBucket < bucketCapacity) { + long offset = getObjectArrayByteOffset(bucketIndex); + Object currentObj = bucket.readObject(Word.unsigned(offset)); + if (currentObj.equals(nullHandle)) { + if (bucket.logicCompareAndSwapObject(Word.unsigned(offset), null, obj, LocationIdentity.ANY_LOCATION)) { + int newSearchIndexInBucket = (indexInBucket + 1 < bucketCapacity) ? (indexInBucket + 1) : indexInBucket; unusedHandleSearchIndex = toIndex(bucketIndex, newSearchIndexInBucket); // (if the next index is in another bucket, we let the next create() // figure it out) @@ -177,23 +191,28 @@ public ObjectHandle create(Object obj) { // last bucket may be smaller newBucketCapacity = getIndexInBucket(maxIndex) + 1; } - Object[] newBucket = new Object[newBucketCapacity]; - Unsafe.getUnsafe().putReferenceVolatile(newBucket, getObjectArrayByteOffset(0), obj); - if (Unsafe.getUnsafe().compareAndSetReference(buckets, getObjectArrayByteOffset(newBucketIndex), null, newBucket)) { + Pointer newBucketBasePtr = NullableNativeMemory.malloc(Word.unsigned(newBucketCapacity * wordSize), NmtCategory.JNI); + newBucketBasePtr.writeObject(Word.unsigned(0), obj); + + buckets[indexInBucket] = (WordPointer) newBucketBasePtr; + bucketCapacities[indexInBucket] = newBucketCapacity; + +// long newBucketOffset = getObjectArrayByteOffset(newBucketIndex); +// if (bucketsBasePtr.logicCompareAndSwapObject(Word.unsigned(newBucketOffset), null, newBucket, LocationIdentity.ANY_LOCATION)) { unusedHandleSearchIndex = toIndex(newBucketIndex, 1); return toHandle(newBucketIndex, 0); - } +// } // start over: another thread has raced us to create another bucket and won - continue outer; +// continue outer; } } bucketIndex++; - bucket = getBucket(bucketIndex); + bucket = (Pointer)getBucket(bucketIndex); if (bucket == null) { lastExistingBucketIndex = bucketIndex - 1; bucketIndex = 0; - bucket = getBucket(bucketIndex); + bucket = (Pointer)getBucket(bucketIndex); } indexInBucket = 0; } @@ -215,12 +234,12 @@ private Object doGet(ObjectHandle handle) { throw new IllegalArgumentException("Invalid handle"); } long index = toIndex(handle); - Object[] bucket = getBucket(getBucketIndex(index)); - if (bucket == null) { + WordPointer bucket = getBucket(getBucketIndex(index)); + if (bucket.isNull()) { throw new IllegalArgumentException("Invalid handle"); } int indexInBucket = getIndexInBucket(index); - return Unsafe.getUnsafe().getReferenceVolatile(bucket, getObjectArrayByteOffset(indexInBucket)); + return ((Pointer)bucket).readObject(Word.unsigned(getObjectArrayByteOffset(indexInBucket))); } @Override @@ -232,22 +251,26 @@ public void destroy(ObjectHandle handle) { throw new IllegalArgumentException("Invalid handle"); } long index = toIndex(handle); - Object[] bucket = getBucket(getBucketIndex(index)); - if (bucket == null) { + WordPointer bucket = getBucket(getBucketIndex(index)); + if (bucket.isNull()) { throw new IllegalArgumentException("Invalid handle"); } int indexInBucket = getIndexInBucket(index); - Unsafe.getUnsafe().putReferenceRelease(bucket, getObjectArrayByteOffset(indexInBucket), null); + ((Pointer)bucket).writeObject(Word.unsigned(getObjectArrayByteOffset(indexInBucket)), null); } public long computeCurrentCount() { long count = 0; int bucketIndex = 0; - Object[] bucket = getBucket(bucketIndex); - while (bucket != null) { - for (int i = 0; i < bucket.length; i++) { - if (bucket[i] != null) { + long offset = 0; + Object currentObj; + WordPointer bucket = getBucket(bucketIndex); + while (!bucket.isNull()) { + for (int i = 0; i < bucketCapacities[bucketIndex]; i++) { + offset = getObjectArrayByteOffset(i); + currentObj = ((Pointer)bucket).readObject(Word.unsigned(offset)); + if (!currentObj.equals(nullHandle)) { count++; } } @@ -260,12 +283,12 @@ public long computeCurrentCount() { public long computeCurrentCapacity() { long capacity = 0; int bucketIndex = 0; - Object[] bucket = getBucket(bucketIndex); - while (bucket != null) { - capacity += bucket.length; + WordPointer bucket = getBucket(bucketIndex); + while (!bucket.isNull()) { + capacity += bucketCapacities[bucketIndex]; bucketIndex++; bucket = getBucket(bucketIndex); } return capacity; } -} +} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java index 92910e55f4b0..f3f0c0c12ada 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java @@ -34,6 +34,7 @@ import com.oracle.svm.core.RuntimeAssertionsSupport; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.handles.ObjectHandlesImpl; import com.oracle.svm.core.handles.ThreadLocalHandles; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.jni.headers.JNIObjectHandle; @@ -242,7 +243,7 @@ public static JNIObjectHandle newGlobalRef(JNIObjectHandle handle) { result = JNIImageHeapHandles.toGlobal(handle); } else { Object obj = getObject(handle); - if (obj != null) { + if (!obj.equals(nullHandle())) { result = JNIGlobalHandles.create(obj); } } @@ -261,7 +262,7 @@ public static JNIObjectHandle newWeakGlobalRef(JNIObjectHandle handle) { result = JNIImageHeapHandles.toWeakGlobal(handle); } else { Object obj = getObject(handle); - if (obj != null) { + if (!obj.equals(nullHandle())) { result = JNIGlobalHandles.createWeak(obj); } } @@ -345,7 +346,7 @@ private static Word isolateHash() { * @param handle The raw, unencoded handle to the Java object. * @return The resulting strong JNI object handle with embedded metadata. */ - private static JNIObjectHandle encode(ObjectHandle handle) { + private static JNIObjectHandle encodeStrong(ObjectHandle handle) { SignedWord h = (Word) handle; if (JNIObjectHandles.haveAssertions()) { assert h.and(HANDLE_BITS_MASK).equal(h) : "unencoded handle must fit in range"; @@ -385,11 +386,13 @@ private static ObjectHandle decode(JNIObjectHandle handle) { static T getObject(JNIObjectHandle handle) { SignedWord handleValue = (Word) handle; - if ((handleValue.toLong() & WEAK_HANDLE_FLAG.toLong()) == 0) { + if (handleValue.greaterOrEqual(STRONG_GLOBAL_RANGE_MIN) && + handleValue.lessOrEqual(STRONG_GLOBAL_RANGE_MAX)) { return strongGlobalHandles.get(decode(handle)); } - if ((handleValue.toLong() & WEAK_HANDLE_FLAG.toLong()) == 1) { + if (handleValue.greaterOrEqual(WEAK_GLOBAL_RANGE_MIN) && + handleValue.lessOrEqual(WEAK_GLOBAL_RANGE_MAX)) { return weakGlobalHandles.get(decode((handle))); } @@ -398,18 +401,20 @@ static T getObject(JNIObjectHandle handle) { static JNIObjectRefType getHandleType(JNIObjectHandle handle) { SignedWord handleValue = (Word) handle; - if ((handleValue.toLong() & WEAK_HANDLE_FLAG.toLong()) == 0) { + if (handleValue.greaterOrEqual(STRONG_GLOBAL_RANGE_MIN) && + handleValue.lessOrEqual(STRONG_GLOBAL_RANGE_MAX)) { return JNIObjectRefType.Global; } - if ((handleValue.toLong() & WEAK_HANDLE_FLAG.toLong()) == 1) { + if (handleValue.greaterOrEqual(WEAK_GLOBAL_RANGE_MIN) && + handleValue.lessOrEqual(WEAK_GLOBAL_RANGE_MAX)) { return JNIObjectRefType.WeakGlobal; } return JNIObjectRefType.Invalid; } static JNIObjectHandle create(Object obj) { - return encode(strongGlobalHandles.create(obj)); + return encodeStrong(strongGlobalHandles.create(obj)); } static void destroy(JNIObjectHandle handle) { From 44ec6e3b218a1866215fc7b7c80950714807eb3a Mon Sep 17 00:00:00 2001 From: asselyam Date: Sat, 15 Nov 2025 13:56:12 +0100 Subject: [PATCH 03/10] Update ObjectHandlesImpl memory allocation --- .../svm/core/handles/ObjectHandlesImpl.java | 88 ++++++++++++++----- 1 file changed, 65 insertions(+), 23 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java index ce9989f34f5c..0acefba974e2 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java @@ -24,8 +24,10 @@ */ package com.oracle.svm.core.handles; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.memory.NullableNativeMemory; import com.oracle.svm.core.nmt.NmtCategory; +import org.graalvm.nativeimage.ImageInfo; import org.graalvm.nativeimage.ObjectHandle; import org.graalvm.nativeimage.ObjectHandles; import org.graalvm.nativeimage.c.type.WordPointer; @@ -55,7 +57,6 @@ public final class ObjectHandlesImpl implements ObjectHandles { private static final int MAX_FIRST_BUCKET_CAPACITY = 1024; - private static final int wordSize = ConfigurationValues.getTarget().wordSize; static { // must be a power of 2 for the arithmetic below to work assert Integer.lowestOneBit(MAX_FIRST_BUCKET_CAPACITY) == MAX_FIRST_BUCKET_CAPACITY; } @@ -67,6 +68,7 @@ public final class ObjectHandlesImpl implements ObjectHandles { private final WordPointer[] buckets; private final int[] bucketCapacities; private volatile long unusedHandleSearchIndex = 0; + private int deferredFirstBucketCapacity = -1; public ObjectHandlesImpl() { this(Word.signed(1), Word.signed(Long.MAX_VALUE), Word.signed(0)); @@ -90,9 +92,7 @@ public ObjectHandlesImpl(SignedWord rangeMin, SignedWord rangeMax, SignedWord nu firstBucketCapacity = lastBucketCapacity; } - Pointer bucketBasePtr = NullableNativeMemory.malloc(Word.unsigned(firstBucketCapacity * wordSize), NmtCategory.JNI); - buckets[0] = (WordPointer) bucketBasePtr; - bucketCapacities[0] = firstBucketCapacity; + deferredFirstBucketCapacity = firstBucketCapacity; } public boolean isInRange(ObjectHandle handle) { @@ -126,13 +126,38 @@ private static int getIndexInBucket(long index) { } private static long getObjectArrayByteOffset(int index) { - return (long) index * wordSize; + return (long) index * ConfigurationValues.getTarget().wordSize; + } + + @Uninterruptible(reason = "Called from critical sections") + private static WordPointer allocateBucket(int capacity) { + if (ImageInfo.inImageCode()) { + return null; + } + long bytes = (long) capacity * ConfigurationValues.getTarget().wordSize; + Pointer ptr = NullableNativeMemory.malloc(Word.unsigned(bytes), NmtCategory.JNI); + return (WordPointer) ptr; } private WordPointer getBucket(int bucketIndex) { // buckets[i] is changed only once from null to its final value: try without volatile first WordPointer bucket = buckets[bucketIndex]; + if (bucket != null) { + return bucket; + } + if (bucketIndex == 0 && deferredFirstBucketCapacity != -1) { + // This is the first time we are accessing the capacity at runtime. + bucketCapacities[0] = deferredFirstBucketCapacity; + deferredFirstBucketCapacity = -1; // Mark as initialized + } + + if (ImageInfo.inImageCode()) { + return null; + } + + bucket = allocateBucket(bucketCapacities[bucketIndex]); + buckets[bucketIndex] = bucket; return bucket; } @@ -162,9 +187,12 @@ public ObjectHandle create(Object obj) { for (;;) { while (indexInBucket < bucketCapacity) { - long offset = getObjectArrayByteOffset(bucketIndex); + long offset = getObjectArrayByteOffset(indexInBucket); + if (bucket == null) { + throw new IllegalStateException("Bucket not allocated"); + } Object currentObj = bucket.readObject(Word.unsigned(offset)); - if (currentObj.equals(nullHandle)) { + if (currentObj == null) { if (bucket.logicCompareAndSwapObject(Word.unsigned(offset), null, obj, LocationIdentity.ANY_LOCATION)) { int newSearchIndexInBucket = (indexInBucket + 1 < bucketCapacity) ? (indexInBucket + 1) : indexInBucket; unusedHandleSearchIndex = toIndex(bucketIndex, newSearchIndexInBucket); @@ -191,19 +219,30 @@ public ObjectHandle create(Object obj) { // last bucket may be smaller newBucketCapacity = getIndexInBucket(maxIndex) + 1; } - Pointer newBucketBasePtr = NullableNativeMemory.malloc(Word.unsigned(newBucketCapacity * wordSize), NmtCategory.JNI); - newBucketBasePtr.writeObject(Word.unsigned(0), obj); - buckets[indexInBucket] = (WordPointer) newBucketBasePtr; - bucketCapacities[indexInBucket] = newBucketCapacity; + // Allocate new bucket memory manually + WordPointer newBucket = allocateBucket(newBucketCapacity); + if (newBucket == null) { + // Allocation failed at build time or due to error (shouldn't happen at runtime now) + continue outer; + } + Pointer newBucketPtr = (Pointer) newBucket; + // Initialize the first slot with the object + newBucketPtr.writeObject(Word.unsigned(0), obj); + + // CAS-insert bucket pointer into `buckets[newBucketIndex]` + if (buckets[newBucketIndex] == null) { + buckets[newBucketIndex] = newBucket; + bucketCapacities[newBucketIndex] = newBucketCapacity; -// long newBucketOffset = getObjectArrayByteOffset(newBucketIndex); -// if (bucketsBasePtr.logicCompareAndSwapObject(Word.unsigned(newBucketOffset), null, newBucket, LocationIdentity.ANY_LOCATION)) { unusedHandleSearchIndex = toIndex(newBucketIndex, 1); return toHandle(newBucketIndex, 0); -// } - // start over: another thread has raced us to create another bucket and won -// continue outer; + } else { + if (!ImageInfo.inImageCode()) { + NullableNativeMemory.free(newBucketPtr); + } + continue outer; + } } } @@ -235,8 +274,11 @@ private Object doGet(ObjectHandle handle) { } long index = toIndex(handle); WordPointer bucket = getBucket(getBucketIndex(index)); - if (bucket.isNull()) { - throw new IllegalArgumentException("Invalid handle"); + if (bucket == null) { + if (ImageInfo.inImageCode()) { + return nullHandle; // or skip writing + } + throw new IllegalStateException("Bucket not allocated"); } int indexInBucket = getIndexInBucket(index); return ((Pointer)bucket).readObject(Word.unsigned(getObjectArrayByteOffset(indexInBucket))); @@ -252,8 +294,8 @@ public void destroy(ObjectHandle handle) { } long index = toIndex(handle); WordPointer bucket = getBucket(getBucketIndex(index)); - if (bucket.isNull()) { - throw new IllegalArgumentException("Invalid handle"); + if (bucket == null) { + throw new IllegalStateException("Bucket not allocated"); } int indexInBucket = getIndexInBucket(index); ((Pointer)bucket).writeObject(Word.unsigned(getObjectArrayByteOffset(indexInBucket)), null); @@ -266,11 +308,11 @@ public long computeCurrentCount() { long offset = 0; Object currentObj; WordPointer bucket = getBucket(bucketIndex); - while (!bucket.isNull()) { + while (bucket != null) { for (int i = 0; i < bucketCapacities[bucketIndex]; i++) { offset = getObjectArrayByteOffset(i); currentObj = ((Pointer)bucket).readObject(Word.unsigned(offset)); - if (!currentObj.equals(nullHandle)) { + if (currentObj != null) { count++; } } @@ -284,7 +326,7 @@ public long computeCurrentCapacity() { long capacity = 0; int bucketIndex = 0; WordPointer bucket = getBucket(bucketIndex); - while (!bucket.isNull()) { + while (bucket != null) { capacity += bucketCapacities[bucketIndex]; bucketIndex++; bucket = getBucket(bucketIndex); From 0de090cf93382a3f6b3d5eddcc7c4eec37dd3864 Mon Sep 17 00:00:00 2001 From: asselyam Date: Sun, 23 Nov 2025 21:41:58 +0100 Subject: [PATCH 04/10] Fix native memory handling --- .../svm/core/handles/ObjectHandlesImpl.java | 75 +++++++++---------- .../oracle/svm/core/jni/JNIObjectHandles.java | 4 +- 2 files changed, 38 insertions(+), 41 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java index 0acefba974e2..5ac69d4d6314 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java @@ -39,6 +39,7 @@ import com.oracle.svm.core.config.ConfigurationValues; import jdk.graal.compiler.word.Word; +import org.graalvm.word.WordFactory; /** * This class implements {@link ObjectHandle word}-sized integer handles that refer to Java objects. @@ -85,6 +86,7 @@ public ObjectHandlesImpl(SignedWord rangeMin, SignedWord rangeMax, SignedWord nu int lastBucketIndex = getBucketIndex(maxIndex); int lastBucketCapacity = getIndexInBucket(maxIndex) + 1; buckets = new WordPointer[lastBucketIndex + 1]; + bucketCapacities = new int[lastBucketIndex + 1]; int firstBucketCapacity = MAX_FIRST_BUCKET_CAPACITY; @@ -131,18 +133,22 @@ private static long getObjectArrayByteOffset(int index) { @Uninterruptible(reason = "Called from critical sections") private static WordPointer allocateBucket(int capacity) { - if (ImageInfo.inImageCode()) { - return null; - } long bytes = (long) capacity * ConfigurationValues.getTarget().wordSize; Pointer ptr = NullableNativeMemory.malloc(Word.unsigned(bytes), NmtCategory.JNI); return (WordPointer) ptr; } + /** + * Helper to get the base address of the buckets array data block. + * This uses the standard idiom for accessing the native backing of a Word-based array. + */ + private Pointer getBucketsDataPointer() { + return Word.objectToUntrackedPointer(buckets); + } + private WordPointer getBucket(int bucketIndex) { - // buckets[i] is changed only once from null to its final value: try without volatile first WordPointer bucket = buckets[bucketIndex]; - if (bucket != null) { + if (!bucket.isNull()) { return bucket; } @@ -152,10 +158,6 @@ private WordPointer getBucket(int bucketIndex) { deferredFirstBucketCapacity = -1; // Mark as initialized } - if (ImageInfo.inImageCode()) { - return null; - } - bucket = allocateBucket(bucketCapacities[bucketIndex]); buckets[bucketIndex] = bucket; return bucket; @@ -182,18 +184,16 @@ public ObjectHandle create(Object obj) { int bucketIndex = startBucketIndex; int indexInBucket = startIndexInBucket; int lastExistingBucketIndex = -1; - Pointer bucket = (Pointer) getBucket(bucketIndex); + WordPointer bucket = getBucket(bucketIndex); int bucketCapacity = bucketCapacities[bucketIndex]; for (;;) { while (indexInBucket < bucketCapacity) { long offset = getObjectArrayByteOffset(indexInBucket); - if (bucket == null) { - throw new IllegalStateException("Bucket not allocated"); - } - Object currentObj = bucket.readObject(Word.unsigned(offset)); + Object currentObj = ((Pointer)bucket).readObject(Word.unsigned(offset)); if (currentObj == null) { - if (bucket.logicCompareAndSwapObject(Word.unsigned(offset), null, obj, LocationIdentity.ANY_LOCATION)) { + Object prev = ((Pointer)bucket).compareAndSwapObject(Word.unsigned(offset), null, obj, LocationIdentity.ANY_LOCATION); + if (prev == null) { int newSearchIndexInBucket = (indexInBucket + 1 < bucketCapacity) ? (indexInBucket + 1) : indexInBucket; unusedHandleSearchIndex = toIndex(bucketIndex, newSearchIndexInBucket); // (if the next index is in another bucket, we let the next create() @@ -211,7 +211,7 @@ public ObjectHandle create(Object obj) { throw new IllegalStateException("Handle space exhausted"); } int newBucketIndex = lastExistingBucketIndex + 1; - if (getBucket(newBucketIndex) != null) { + if (getBucket(newBucketIndex).isNonNull()) { continue outer; // start over: another thread has created a new bucket } int newBucketCapacity = (MAX_FIRST_BUCKET_CAPACITY << newBucketIndex); @@ -220,18 +220,13 @@ public ObjectHandle create(Object obj) { newBucketCapacity = getIndexInBucket(maxIndex) + 1; } - // Allocate new bucket memory manually WordPointer newBucket = allocateBucket(newBucketCapacity); - if (newBucket == null) { - // Allocation failed at build time or due to error (shouldn't happen at runtime now) - continue outer; - } - Pointer newBucketPtr = (Pointer) newBucket; - // Initialize the first slot with the object - newBucketPtr.writeObject(Word.unsigned(0), obj); + ((Pointer) newBucket).writeObject(Word.unsigned(0), obj); + + offset = getObjectArrayByteOffset(newBucketIndex); + Object prev = getBucketsDataPointer().compareAndSwapObject(Word.unsigned(offset), Word.nullPointer(), newBucket, LocationIdentity.ANY_LOCATION); - // CAS-insert bucket pointer into `buckets[newBucketIndex]` - if (buckets[newBucketIndex] == null) { + if (((WordPointer)prev).isNull()) { buckets[newBucketIndex] = newBucket; bucketCapacities[newBucketIndex] = newBucketCapacity; @@ -239,7 +234,7 @@ public ObjectHandle create(Object obj) { return toHandle(newBucketIndex, 0); } else { if (!ImageInfo.inImageCode()) { - NullableNativeMemory.free(newBucketPtr); + NullableNativeMemory.free(newBucket); } continue outer; } @@ -247,11 +242,11 @@ public ObjectHandle create(Object obj) { } bucketIndex++; - bucket = (Pointer)getBucket(bucketIndex); - if (bucket == null) { + bucket = getBucket(bucketIndex); + if (bucket.isNull()) { lastExistingBucketIndex = bucketIndex - 1; bucketIndex = 0; - bucket = (Pointer)getBucket(bucketIndex); + bucket = getBucket(bucketIndex); } indexInBucket = 0; } @@ -266,7 +261,7 @@ public T get(ObjectHandle handle) { } private Object doGet(ObjectHandle handle) { - if (handle.equal(nullHandle)) { + if (handle.rawValue() == nullHandle.rawValue()) { return null; } if (!isInRange(handle)) { @@ -274,19 +269,18 @@ private Object doGet(ObjectHandle handle) { } long index = toIndex(handle); WordPointer bucket = getBucket(getBucketIndex(index)); - if (bucket == null) { - if (ImageInfo.inImageCode()) { - return nullHandle; // or skip writing - } + + if (bucket.isNull()) { throw new IllegalStateException("Bucket not allocated"); } + int indexInBucket = getIndexInBucket(index); return ((Pointer)bucket).readObject(Word.unsigned(getObjectArrayByteOffset(indexInBucket))); } @Override public void destroy(ObjectHandle handle) { - if (handle.equal(nullHandle)) { + if (handle.rawValue() == nullHandle.rawValue()) { return; } if (!isInRange(handle)) { @@ -294,9 +288,11 @@ public void destroy(ObjectHandle handle) { } long index = toIndex(handle); WordPointer bucket = getBucket(getBucketIndex(index)); - if (bucket == null) { + + if (bucket.isNull()) { throw new IllegalStateException("Bucket not allocated"); } + int indexInBucket = getIndexInBucket(index); ((Pointer)bucket).writeObject(Word.unsigned(getObjectArrayByteOffset(indexInBucket)), null); } @@ -308,7 +304,7 @@ public long computeCurrentCount() { long offset = 0; Object currentObj; WordPointer bucket = getBucket(bucketIndex); - while (bucket != null) { + while (bucket.isNonNull()) { for (int i = 0; i < bucketCapacities[bucketIndex]; i++) { offset = getObjectArrayByteOffset(i); currentObj = ((Pointer)bucket).readObject(Word.unsigned(offset)); @@ -326,11 +322,12 @@ public long computeCurrentCapacity() { long capacity = 0; int bucketIndex = 0; WordPointer bucket = getBucket(bucketIndex); - while (bucket != null) { + while (bucket.isNonNull()) { capacity += bucketCapacities[bucketIndex]; bucketIndex++; bucket = getBucket(bucketIndex); } return capacity; } + } \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java index f3f0c0c12ada..6af442598f75 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java @@ -243,7 +243,7 @@ public static JNIObjectHandle newGlobalRef(JNIObjectHandle handle) { result = JNIImageHeapHandles.toGlobal(handle); } else { Object obj = getObject(handle); - if (!obj.equals(nullHandle())) { + if (obj != null) { result = JNIGlobalHandles.create(obj); } } @@ -262,7 +262,7 @@ public static JNIObjectHandle newWeakGlobalRef(JNIObjectHandle handle) { result = JNIImageHeapHandles.toWeakGlobal(handle); } else { Object obj = getObject(handle); - if (!obj.equals(nullHandle())) { + if (obj != null) { result = JNIGlobalHandles.createWeak(obj); } } From e74274af7029f858f1fc623acbfa104abd2b2672 Mon Sep 17 00:00:00 2001 From: asselyam Date: Mon, 1 Dec 2025 08:07:00 +0100 Subject: [PATCH 05/10] segmentation fault --- .../oracle/svm/core/genscavenge/GCImpl.java | 31 ++- .../svm/core/handles/ObjectHandlesImpl.java | 178 ++++++++++-------- .../oracle/svm/core/jni/JNIObjectHandles.java | 11 ++ 3 files changed, 137 insertions(+), 83 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index b8254deb6f7b..3fa0ed3b5a35 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -31,6 +31,9 @@ import java.lang.ref.Reference; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.handles.ObjectHandlesImpl; +import com.oracle.svm.core.jni.JNIObjectHandles; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -39,6 +42,7 @@ import org.graalvm.nativeimage.c.struct.RawField; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; @@ -110,7 +114,6 @@ import com.oracle.svm.core.util.TimeUtils; import com.oracle.svm.core.util.Timer; import com.oracle.svm.core.util.VMError; - import jdk.graal.compiler.api.replacements.Fold; import jdk.graal.compiler.word.Word; @@ -136,6 +139,7 @@ public final class GCImpl implements GC { private boolean completeCollection = false; private UnsignedWord collectionEpoch = Word.zero(); private long lastWholeHeapExaminedNanos = -1; + private final GlobalHandlesBucketCallback globalHandlesCallback = new GlobalHandlesBucketCallback(greyToBlackObjRefVisitor);; @Platforms(Platform.HOSTED_ONLY.class) GCImpl() { @@ -643,6 +647,29 @@ private void scan(boolean incremental) { } } + static final class GlobalHandlesBucketCallback implements ObjectHandlesImpl.BucketCallback { + + private final GreyToBlackObjRefVisitor visitor; + + @Uninterruptible(reason = "Construct callback with visitor") + GlobalHandlesBucketCallback(GreyToBlackObjRefVisitor visitor) { + this.visitor = visitor; + } + + @Override + @Uninterruptible(reason = "Construct callback with visitor") + public void visitBucket(Pointer bucketAddress, int count) { + visitor.visitObjectReferences( + bucketAddress, + false, + ConfigurationValues.getTarget().wordSize, + null, + count + ); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void scanFromRoots() { Timer scanFromRootsTimer = timers.scanFromRoots.start(); @@ -674,6 +701,7 @@ private void scanFromRoots() { try { blackenStackRoots(); blackenThreadLocals(); + JNIObjectHandles.scanGlobalHandleBuckets(globalHandlesCallback); blackenImageHeapRoots(); blackenMetaspace(); } finally { @@ -738,6 +766,7 @@ private void scanFromDirtyRoots() { blackenDirtyCardRoots(); blackenStackRoots(); blackenThreadLocals(); + JNIObjectHandles.scanGlobalHandleBuckets(globalHandlesCallback); blackenDirtyImageHeapRoots(); blackenMetaspace(); } finally { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java index 5ac69d4d6314..d1136a5d11de 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java @@ -25,9 +25,9 @@ package com.oracle.svm.core.handles; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.memory.NativeMemory; import com.oracle.svm.core.memory.NullableNativeMemory; import com.oracle.svm.core.nmt.NmtCategory; -import org.graalvm.nativeimage.ImageInfo; import org.graalvm.nativeimage.ObjectHandle; import org.graalvm.nativeimage.ObjectHandles; import org.graalvm.nativeimage.c.type.WordPointer; @@ -35,12 +35,13 @@ import org.graalvm.word.Pointer; import org.graalvm.word.SignedWord; import org.graalvm.word.WordBase; - import com.oracle.svm.core.config.ConfigurationValues; - import jdk.graal.compiler.word.Word; import org.graalvm.word.WordFactory; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; + /** * This class implements {@link ObjectHandle word}-sized integer handles that refer to Java objects. * {@link #create(Object) Creating}, {@link #get(ObjectHandle) dereferencing} and @@ -58,6 +59,7 @@ public final class ObjectHandlesImpl implements ObjectHandles { private static final int MAX_FIRST_BUCKET_CAPACITY = 1024; + static { // must be a power of 2 for the arithmetic below to work assert Integer.lowestOneBit(MAX_FIRST_BUCKET_CAPACITY) == MAX_FIRST_BUCKET_CAPACITY; } @@ -66,10 +68,9 @@ public final class ObjectHandlesImpl implements ObjectHandles { private final SignedWord rangeMax; private final SignedWord nullHandle; - private final WordPointer[] buckets; - private final int[] bucketCapacities; + private final Pointer[] buckets; + private final int[] capacities; private volatile long unusedHandleSearchIndex = 0; - private int deferredFirstBucketCapacity = -1; public ObjectHandlesImpl() { this(Word.signed(1), Word.signed(Long.MAX_VALUE), Word.signed(0)); @@ -85,16 +86,34 @@ public ObjectHandlesImpl(SignedWord rangeMin, SignedWord rangeMax, SignedWord nu long maxIndex = toIndex(rangeMax); int lastBucketIndex = getBucketIndex(maxIndex); int lastBucketCapacity = getIndexInBucket(maxIndex) + 1; - buckets = new WordPointer[lastBucketIndex + 1]; - - bucketCapacities = new int[lastBucketIndex + 1]; + buckets = new Pointer[lastBucketIndex + 1]; + capacities = new int[lastBucketIndex + 1]; int firstBucketCapacity = MAX_FIRST_BUCKET_CAPACITY; - if (lastBucketIndex == 0) { // if our range is small, we may have only a single small bucket firstBucketCapacity = lastBucketCapacity; } + capacities[0] = firstBucketCapacity; + + Pointer nullPtr = WordFactory.nullPointer(); + for (int i = 0; i < buckets.length; i++) { + buckets[i] = nullPtr; + } + } - deferredFirstBucketCapacity = firstBucketCapacity; + public interface BucketCallback { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void visitBucket(Pointer bucketAddress, int capacity); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void scanAllBuckets(BucketCallback callback) { + for (int i = 0; i < buckets.length; i++) { + Pointer bucket = getBucket(i); + if (bucket.isNull()) { + continue; + } + callback.visitBucket(bucket, capacities[i]); + } } public boolean isInRange(ObjectHandle handle) { @@ -127,40 +146,31 @@ private static int getIndexInBucket(long index) { return Math.toIntExact(bucketBit ^ displacedIndex); } - private static long getObjectArrayByteOffset(int index) { - return (long) index * ConfigurationValues.getTarget().wordSize; + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Pointer getBucket(int bucketIndex) { + // buckets[i] is changed only once from null to its final value: try without volatile first + Pointer bucket = buckets[bucketIndex]; + return bucket; } - @Uninterruptible(reason = "Called from critical sections") - private static WordPointer allocateBucket(int capacity) { - long bytes = (long) capacity * ConfigurationValues.getTarget().wordSize; - Pointer ptr = NullableNativeMemory.malloc(Word.unsigned(bytes), NmtCategory.JNI); - return (WordPointer) ptr; + private static int wordSize() { + return ConfigurationValues.getTarget().wordSize; } - /** - * Helper to get the base address of the buckets array data block. - * This uses the standard idiom for accessing the native backing of a Word-based array. - */ - private Pointer getBucketsDataPointer() { - return Word.objectToUntrackedPointer(buckets); + private static Pointer slotPointer(Pointer bucket, int indexInBucket) { + return bucket.add(indexInBucket * wordSize()); } - private WordPointer getBucket(int bucketIndex) { - WordPointer bucket = buckets[bucketIndex]; - if (!bucket.isNull()) { - return bucket; - } + private static Object readSlot(Pointer bucket, int indexInBucket) { + return slotPointer(bucket, indexInBucket).readObject(0); + } - if (bucketIndex == 0 && deferredFirstBucketCapacity != -1) { - // This is the first time we are accessing the capacity at runtime. - bucketCapacities[0] = deferredFirstBucketCapacity; - deferredFirstBucketCapacity = -1; // Mark as initialized - } + private static void writeSlot(Pointer bucket, int indexInBucket, Object value) { + slotPointer(bucket, indexInBucket).writeObject(0, value); + } - bucket = allocateBucket(bucketCapacities[bucketIndex]); - buckets[bucketIndex] = bucket; - return bucket; + private static boolean casSlot(Pointer bucket, int indexInBucket, Object expected, Object value) { + return slotPointer(bucket, indexInBucket).logicCompareAndSwapObject(0, expected, value, LocationIdentity.ANY_LOCATION); } @Override @@ -176,6 +186,15 @@ public ObjectHandle create(Object obj) { return (ObjectHandle) nullHandle; } + if (buckets[0].isNull()) { + synchronized (this) { + if (buckets[0].isNull()) { + Pointer p = NullableNativeMemory.calloc(capacities[0] * wordSize(), NmtCategory.JNI); + buckets[0] = p; + } + } + } + outer: for (;;) { long startIndex = unusedHandleSearchIndex; int startBucketIndex = getBucketIndex(startIndex); @@ -184,17 +203,14 @@ public ObjectHandle create(Object obj) { int bucketIndex = startBucketIndex; int indexInBucket = startIndexInBucket; int lastExistingBucketIndex = -1; - WordPointer bucket = getBucket(bucketIndex); - int bucketCapacity = bucketCapacities[bucketIndex]; + Pointer bucket = getBucket(bucketIndex); for (;;) { - while (indexInBucket < bucketCapacity) { - long offset = getObjectArrayByteOffset(indexInBucket); - Object currentObj = ((Pointer)bucket).readObject(Word.unsigned(offset)); - if (currentObj == null) { - Object prev = ((Pointer)bucket).compareAndSwapObject(Word.unsigned(offset), null, obj, LocationIdentity.ANY_LOCATION); - if (prev == null) { - int newSearchIndexInBucket = (indexInBucket + 1 < bucketCapacity) ? (indexInBucket + 1) : indexInBucket; + while (indexInBucket < capacities[bucketIndex]) { + Object current = readSlot(bucket, indexInBucket); + if (current == null) { + if (casSlot(bucket, indexInBucket, null, obj)) { + int newSearchIndexInBucket = (indexInBucket + 1 < capacities[bucketIndex]) ? (indexInBucket + 1) : indexInBucket; unusedHandleSearchIndex = toIndex(bucketIndex, newSearchIndexInBucket); // (if the next index is in another bucket, we let the next create() // figure it out) @@ -210,34 +226,32 @@ public ObjectHandle create(Object obj) { if (lastExistingBucketIndex == getBucketIndex(maxIndex)) { throw new IllegalStateException("Handle space exhausted"); } + int newBucketIndex = lastExistingBucketIndex + 1; - if (getBucket(newBucketIndex).isNonNull()) { + if (!getBucket(newBucketIndex).isNull()) { continue outer; // start over: another thread has created a new bucket } + int newBucketCapacity = (MAX_FIRST_BUCKET_CAPACITY << newBucketIndex); if (newBucketIndex == getBucketIndex(maxIndex)) { // last bucket may be smaller newBucketCapacity = getIndexInBucket(maxIndex) + 1; } - WordPointer newBucket = allocateBucket(newBucketCapacity); - ((Pointer) newBucket).writeObject(Word.unsigned(0), obj); - - offset = getObjectArrayByteOffset(newBucketIndex); - Object prev = getBucketsDataPointer().compareAndSwapObject(Word.unsigned(offset), Word.nullPointer(), newBucket, LocationIdentity.ANY_LOCATION); - - if (((WordPointer)prev).isNull()) { - buckets[newBucketIndex] = newBucket; - bucketCapacities[newBucketIndex] = newBucketCapacity; + Pointer newBucket = NullableNativeMemory.calloc(newBucketCapacity * wordSize(), NmtCategory.JNI); + writeSlot(newBucket, 0, obj); - unusedHandleSearchIndex = toIndex(newBucketIndex, 1); - return toHandle(newBucketIndex, 0); - } else { - if (!ImageInfo.inImageCode()) { - NullableNativeMemory.free(newBucket); + synchronized (this) { + if (buckets[newBucketIndex].isNull()) { + buckets[newBucketIndex] = newBucket; + capacities[newBucketIndex] = newBucketCapacity; + unusedHandleSearchIndex = toIndex(newBucketIndex, 1); + return toHandle(newBucketIndex, 0); } - continue outer; } + + NullableNativeMemory.free(newBucket); + continue outer; } } @@ -261,54 +275,55 @@ public T get(ObjectHandle handle) { } private Object doGet(ObjectHandle handle) { - if (handle.rawValue() == nullHandle.rawValue()) { + if (handle.equal(nullHandle)) { return null; } if (!isInRange(handle)) { throw new IllegalArgumentException("Invalid handle"); } + long index = toIndex(handle); - WordPointer bucket = getBucket(getBucketIndex(index)); + int bucketIndex = getBucketIndex(index); + int indexInBucket = getIndexInBucket(index); + + Pointer bucket = getBucket(bucketIndex); if (bucket.isNull()) { throw new IllegalStateException("Bucket not allocated"); } - int indexInBucket = getIndexInBucket(index); - return ((Pointer)bucket).readObject(Word.unsigned(getObjectArrayByteOffset(indexInBucket))); + return readSlot(bucket, indexInBucket); } @Override public void destroy(ObjectHandle handle) { - if (handle.rawValue() == nullHandle.rawValue()) { + if (handle.equal(nullHandle)) { return; } if (!isInRange(handle)) { throw new IllegalArgumentException("Invalid handle"); } + long index = toIndex(handle); - WordPointer bucket = getBucket(getBucketIndex(index)); + int bucketIndex = getBucketIndex(index); + int indexInBucket = getIndexInBucket(index); + Pointer bucket = getBucket(bucketIndex); if (bucket.isNull()) { throw new IllegalStateException("Bucket not allocated"); } - int indexInBucket = getIndexInBucket(index); - ((Pointer)bucket).writeObject(Word.unsigned(getObjectArrayByteOffset(indexInBucket)), null); + writeSlot(bucket, indexInBucket, null); } public long computeCurrentCount() { long count = 0; int bucketIndex = 0; - long offset = 0; - Object currentObj; - WordPointer bucket = getBucket(bucketIndex); - while (bucket.isNonNull()) { - for (int i = 0; i < bucketCapacities[bucketIndex]; i++) { - offset = getObjectArrayByteOffset(i); - currentObj = ((Pointer)bucket).readObject(Word.unsigned(offset)); - if (currentObj != null) { + Pointer bucket = getBucket(bucketIndex); + while (!bucket.isNull()) { + for (int i = 0; i < capacities[bucketIndex]; i++) { + if (readSlot(bucket, i) != null) { count++; } } @@ -321,13 +336,12 @@ public long computeCurrentCount() { public long computeCurrentCapacity() { long capacity = 0; int bucketIndex = 0; - WordPointer bucket = getBucket(bucketIndex); - while (bucket.isNonNull()) { - capacity += bucketCapacities[bucketIndex]; + Pointer bucket = getBucket(bucketIndex); + while (!bucket.isNull()) { + capacity += capacities[bucketIndex]; bucketIndex++; bucket = getBucket(bucketIndex); } return capacity; } - } \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java index 6af442598f75..084c5de187e5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java @@ -283,6 +283,11 @@ static int getLocalHandleCount() { static long computeCurrentGlobalHandleCount() { return JNIGlobalHandles.computeCurrentCount(); } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void scanGlobalHandleBuckets(ObjectHandlesImpl.BucketCallback callback) { + JNIGlobalHandles.scanAllHandleBuckets(callback); + } } /** @@ -432,6 +437,12 @@ static void destroyWeak(JNIObjectHandle weakRef) { public static long computeCurrentCount() { return strongGlobalHandles.computeCurrentCount() + weakGlobalHandles.computeCurrentCount(); } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void scanAllHandleBuckets(ObjectHandlesImpl.BucketCallback callback) { + strongGlobalHandles.scanAllBuckets(callback); + weakGlobalHandles.scanAllBuckets(callback); + } } /** From 7f985a67e92ddd0392061e1f64a8502d25d4a8b5 Mon Sep 17 00:00:00 2001 From: asselyam Date: Tue, 2 Dec 2025 21:53:39 +0100 Subject: [PATCH 06/10] Update ObjectHandlesImpl.java --- .../com/oracle/svm/core/handles/ObjectHandlesImpl.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java index d1136a5d11de..0c8ac10a0d42 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java @@ -190,6 +190,9 @@ public ObjectHandle create(Object obj) { synchronized (this) { if (buckets[0].isNull()) { Pointer p = NullableNativeMemory.calloc(capacities[0] * wordSize(), NmtCategory.JNI); + for (int i = 0; i < capacities[0]; i++) { + writeSlot(p, i, null); + } buckets[0] = p; } } @@ -239,6 +242,9 @@ public ObjectHandle create(Object obj) { } Pointer newBucket = NullableNativeMemory.calloc(newBucketCapacity * wordSize(), NmtCategory.JNI); + for (int i = 0; i < capacities[newBucketIndex]; i++) { + writeSlot(newBucket, i, null); + } writeSlot(newBucket, 0, obj); synchronized (this) { @@ -344,4 +350,4 @@ public long computeCurrentCapacity() { } return capacity; } -} \ No newline at end of file +} From f3a96d1cfa1dc7bbe4f880310444cfe945210214 Mon Sep 17 00:00:00 2001 From: asselyam Date: Tue, 2 Dec 2025 21:55:43 +0100 Subject: [PATCH 07/10] Update ObjectHandlesImpl.java From a8f99f6a0b679a74fdd5fd20a61b7dd85c857217 Mon Sep 17 00:00:00 2001 From: asselyam Date: Fri, 5 Dec 2025 14:24:58 +0100 Subject: [PATCH 08/10] Update ObjectHandlesImpl.java --- .../oracle/svm/core/handles/ObjectHandlesImpl.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java index 0c8ac10a0d42..2d73327746b4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java @@ -25,12 +25,10 @@ package com.oracle.svm.core.handles; import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.memory.NativeMemory; import com.oracle.svm.core.memory.NullableNativeMemory; import com.oracle.svm.core.nmt.NmtCategory; import org.graalvm.nativeimage.ObjectHandle; import org.graalvm.nativeimage.ObjectHandles; -import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.word.LocationIdentity; import org.graalvm.word.Pointer; import org.graalvm.word.SignedWord; @@ -39,9 +37,6 @@ import jdk.graal.compiler.word.Word; import org.graalvm.word.WordFactory; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.VarHandle; - /** * This class implements {@link ObjectHandle word}-sized integer handles that refer to Java objects. * {@link #create(Object) Creating}, {@link #get(ObjectHandle) dereferencing} and @@ -112,7 +107,11 @@ public void scanAllBuckets(BucketCallback callback) { if (bucket.isNull()) { continue; } - callback.visitBucket(bucket, capacities[i]); + int cap = capacities[i]; + if (cap == 0) { + continue; + } + callback.visitBucket(bucket, cap); } } @@ -242,11 +241,10 @@ public ObjectHandle create(Object obj) { } Pointer newBucket = NullableNativeMemory.calloc(newBucketCapacity * wordSize(), NmtCategory.JNI); - for (int i = 0; i < capacities[newBucketIndex]; i++) { + for (int i = 0; i < newBucketCapacity; i++) { writeSlot(newBucket, i, null); } writeSlot(newBucket, 0, obj); - synchronized (this) { if (buckets[newBucketIndex].isNull()) { buckets[newBucketIndex] = newBucket; From 61ebbcf906f4e34a0fd9af414118921b60b0ff26 Mon Sep 17 00:00:00 2001 From: asselyam Date: Fri, 5 Dec 2025 14:25:25 +0100 Subject: [PATCH 09/10] Update JNIObjectHandles.java --- .../oracle/svm/core/jni/JNIObjectHandles.java | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java index 084c5de187e5..3eae294680e3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java @@ -286,7 +286,7 @@ static long computeCurrentGlobalHandleCount() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void scanGlobalHandleBuckets(ObjectHandlesImpl.BucketCallback callback) { - JNIGlobalHandles.scanAllHandleBuckets(callback); + JNIGlobalHandles.scanStrongHandleBuckets(callback); } } @@ -390,32 +390,25 @@ private static ObjectHandle decode(JNIObjectHandle handle) { } static T getObject(JNIObjectHandle handle) { - SignedWord handleValue = (Word) handle; - if (handleValue.greaterOrEqual(STRONG_GLOBAL_RANGE_MIN) && - handleValue.lessOrEqual(STRONG_GLOBAL_RANGE_MAX)) { - return strongGlobalHandles.get(decode(handle)); + if (!isInRange(handle)) { + throw new IllegalArgumentException("Invalid handle"); } - - if (handleValue.greaterOrEqual(WEAK_GLOBAL_RANGE_MIN) && - handleValue.lessOrEqual(WEAK_GLOBAL_RANGE_MAX)) { - return weakGlobalHandles.get(decode((handle))); + SignedWord h = (SignedWord) handle; + if (h.and(WEAK_HANDLE_FLAG).equal(Word.zero())) { + return strongGlobalHandles.get(decode(handle)); + } else { + return weakGlobalHandles.get(decode(handle)); } - - throw new IllegalArgumentException("Invalid handle"); } static JNIObjectRefType getHandleType(JNIObjectHandle handle) { SignedWord handleValue = (Word) handle; - if (handleValue.greaterOrEqual(STRONG_GLOBAL_RANGE_MIN) && - handleValue.lessOrEqual(STRONG_GLOBAL_RANGE_MAX)) { + if (!isInRange(handle)) return JNIObjectRefType.Invalid; + if ((handleValue.and(WEAK_HANDLE_FLAG).equal(Word.zero()))) { return JNIObjectRefType.Global; - } - - if (handleValue.greaterOrEqual(WEAK_GLOBAL_RANGE_MIN) && - handleValue.lessOrEqual(WEAK_GLOBAL_RANGE_MAX)) { + } else { return JNIObjectRefType.WeakGlobal; } - return JNIObjectRefType.Invalid; } static JNIObjectHandle create(Object obj) { @@ -439,8 +432,12 @@ public static long computeCurrentCount() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void scanAllHandleBuckets(ObjectHandlesImpl.BucketCallback callback) { + public static void scanStrongHandleBuckets(ObjectHandlesImpl.BucketCallback callback) { strongGlobalHandles.scanAllBuckets(callback); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void scanWeakHandleBuckets(ObjectHandlesImpl.BucketCallback callback) { weakGlobalHandles.scanAllBuckets(callback); } } From 0b1ed565c5e45271f6ddd1bc8f91de00b900553c Mon Sep 17 00:00:00 2001 From: asselyam Date: Fri, 5 Dec 2025 14:25:58 +0100 Subject: [PATCH 10/10] Update GCImpl.java --- .../oracle/svm/core/genscavenge/GCImpl.java | 239 ++++++++++-------- 1 file changed, 127 insertions(+), 112 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index f3fc7dad5770..4925ebad1976 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -90,7 +90,6 @@ import com.oracle.svm.core.heap.UninterruptibleObjectReferenceVisitor; import com.oracle.svm.core.heap.UninterruptibleObjectVisitor; import com.oracle.svm.core.heap.VMOperationInfos; -import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; import com.oracle.svm.core.interpreter.InterpreterSupport; import com.oracle.svm.core.jdk.RuntimeSupport; import com.oracle.svm.core.jfr.JfrGCWhen; @@ -112,11 +111,6 @@ import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.VMThreadLocalSupport; -import com.oracle.svm.core.traits.BuiltinTraits.AllAccess; -import com.oracle.svm.core.traits.BuiltinTraits.NoLayeredCallbacks; -import com.oracle.svm.core.traits.BuiltinTraits.PartiallyLayerAware; -import com.oracle.svm.core.traits.SingletonLayeredInstallationKind.Independent; -import com.oracle.svm.core.traits.SingletonTraits; import com.oracle.svm.core.util.TimeUtils; import com.oracle.svm.core.util.Timer; import com.oracle.svm.core.util.VMError; @@ -126,7 +120,6 @@ /** * Garbage collector (incremental or complete) for {@link HeapImpl}. */ -@SingletonTraits(access = AllAccess.class, layeredCallbacks = NoLayeredCallbacks.class, layeredInstallationKind = Independent.class, other = PartiallyLayerAware.class) public final class GCImpl implements GC { private static final long K = 1024; static final long M = K * K; @@ -144,17 +137,15 @@ public final class GCImpl implements GC { private final CollectionPolicy policy; private boolean completeCollection = false; - private boolean outOfMemoryCollection = false; private UnsignedWord collectionEpoch = Word.zero(); private long lastWholeHeapExaminedNanos = -1; - private final GlobalHandlesBucketCallback globalHandlesCallback = new GlobalHandlesBucketCallback(greyToBlackObjRefVisitor);; + private static final int WORD_SIZE = ConfigurationValues.getTarget().wordSize; + private final GlobalHandlesBucketCallback GLOBAL_CALLBACK = new GlobalHandlesBucketCallback(greyToBlackObjRefVisitor); @Platforms(Platform.HOSTED_ONLY.class) GCImpl() { this.policy = CollectionPolicy.getInitialPolicy(); - if (ImageLayerBuildingSupport.firstImageBuild()) { - RuntimeSupport.getRuntimeSupport().addShutdownHook(_ -> printGCSummary()); - } + RuntimeSupport.getRuntimeSupport().addShutdownHook(_ -> printGCSummary()); } @Override @@ -313,12 +304,13 @@ private boolean collectImpl(GCCause cause, long beginNanoTime, boolean forceFull try { outOfMemory = doCollectImpl(cause, beginNanoTime, forceFullGC, false); if (outOfMemory) { - outOfMemoryCollection = true; // increase eagerness to free memory + // Avoid running out of memory with a full GC that reclaims softly reachable objects + ReferenceObjectProcessing.setSoftReferencesAreWeak(true); try { verifyHeap(During); outOfMemory = doCollectImpl(cause, System.nanoTime(), true, true); } finally { - outOfMemoryCollection = false; + ReferenceObjectProcessing.setSoftReferencesAreWeak(false); } } } finally { @@ -332,7 +324,7 @@ private boolean doCollectImpl(GCCause cause, long initialBeginNanoTime, boolean ChunkBasedCommittedMemoryProvider.get().beforeGarbageCollection(); - boolean incremental = !forceNoIncremental && !policy.shouldCollectCompletely(false, forceFullGC); + boolean incremental = !forceNoIncremental && !policy.shouldCollectCompletely(false); boolean outOfMemory = false; if (incremental) { @@ -343,7 +335,7 @@ private boolean doCollectImpl(GCCause cause, long initialBeginNanoTime, boolean JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Incremental GC", startTicks); } } - if (!incremental || outOfMemory || forceFullGC || policy.shouldCollectCompletely(incremental, forceFullGC)) { + if (!incremental || outOfMemory || forceFullGC || policy.shouldCollectCompletely(incremental)) { long beginNanoTime = initialBeginNanoTime; if (incremental) { beginNanoTime = System.nanoTime(); @@ -373,7 +365,7 @@ private boolean doCollectOnce(GCCause cause, long beginNanoTime, boolean complet accounting.beforeCollectOnce(completeCollection); policy.onCollectionBegin(completeCollection, beginNanoTime); - doCollectCore(); + doCollectCore(!complete); if (complete) { lastWholeHeapExaminedNanos = System.nanoTime(); } @@ -438,7 +430,7 @@ public static UnsignedWord getChunkBytes() { } private static void resizeAllTlabs() { - if (SubstrateGCOptions.ResizeTLAB.getValue()) { + if (SubstrateGCOptions.TlabOptions.ResizeTLAB.getValue()) { for (IsolateThread thread = VMThreads.firstThread(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) { TlabSupport.resize(thread); } @@ -530,24 +522,13 @@ public void collectCompletely(GCCause cause) { collect(cause, true); } - @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isCompleteCollection() { return completeCollection; } - /** - * Whether the current collection is intended to be more aggressive as a last resort to avoid an - * out of memory condition. - */ - @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public boolean isOutOfMemoryCollection() { - return outOfMemoryCollection; - } - /** Collect, either incrementally or completely, and process discovered references. */ - private void doCollectCore() { + private void doCollectCore(boolean incremental) { GreyToBlackObjRefVisitor.Counters counters = greyToBlackObjRefVisitor.openCounters(); long startTicks; try { @@ -556,15 +537,15 @@ private void doCollectCore() { startTicks = JfrGCEvents.startGCPhasePause(); try { /* Scan reachable objects and potentially already copy them once discovered. */ - scan(); + scan(incremental); } finally { - JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), completeCollection ? "Scan" : "Incremental Scan", startTicks); + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), incremental ? "Incremental Scan" : "Scan", startTicks); } } finally { rootScanTimer.stop(); } - if (completeCollection) { + if (!incremental) { /* Sweep or compact objects in the old generation unless already done by copying. */ HeapImpl.getHeapImpl().getOldGeneration().sweepAndCompact(timers, chunkReleaser); } @@ -614,7 +595,7 @@ private void doCollectCore() { * chunks for copying live old objects with fewer chunk allocations. In either * case, excess chunks are released later. */ - boolean keepAllAlignedChunks = !SerialGCOptions.useCompactingOldGen() && !completeCollection; + boolean keepAllAlignedChunks = !SerialGCOptions.useCompactingOldGen() && incremental; chunkReleaser.release(keepAllAlignedChunks); } finally { JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Release Spaces", startTicks); @@ -659,42 +640,49 @@ private void cleanRuntimeCodeCache() { } @Uninterruptible(reason = "We don't want any safepoint checks in the core part of the GC.") - private void scan() { - Timer timer = completeCollection ? timers.scanFromRoots : timers.scanFromDirtyRoots; - timer.start(); - try { + private void scan(boolean incremental) { + if (incremental) { + scanFromDirtyRoots(); + } else { scanFromRoots(); - } finally { - timer.stop(); } } static final class GlobalHandlesBucketCallback implements ObjectHandlesImpl.BucketCallback { + GreyToBlackObjRefVisitor v; - private final GreyToBlackObjRefVisitor visitor; - - @Uninterruptible(reason = "Construct callback with visitor") - GlobalHandlesBucketCallback(GreyToBlackObjRefVisitor visitor) { - this.visitor = visitor; + private GlobalHandlesBucketCallback(GreyToBlackObjRefVisitor v) { + this.v = v; } @Override @Uninterruptible(reason = "Construct callback with visitor") public void visitBucket(Pointer bucketAddress, int count) { - visitor.visitObjectReferences( + v.visitObjectReferences( bucketAddress, false, - ConfigurationValues.getTarget().wordSize, + WORD_SIZE, null, count ); } } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void scanFromRoots() { - long startTicks = JfrGCEvents.startGCPhasePause(); + Timer scanFromRootsTimer = timers.scanFromRoots.start(); + try { + long startTicks = JfrGCEvents.startGCPhasePause(); + try { + /* + * Snapshot the heap so that objects that are promoted afterwards can be visited. + * When using a compacting old generation, it absorbs all chunks from the young + * generation at this point. + */ + beginPromotion(false); + } finally { + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Snapshot Heap", startTicks); + } startTicks = JfrGCEvents.startGCPhasePause(); try { @@ -711,7 +699,7 @@ private void scanFromRoots() { try { blackenStackRoots(); blackenThreadLocals(); - JNIObjectHandles.scanGlobalHandleBuckets(globalHandlesCallback); + JNIObjectHandles.scanGlobalHandleBuckets(GLOBAL_CALLBACK); blackenImageHeapRoots(); blackenMetaspace(); } finally { @@ -736,23 +724,38 @@ private void scanFromRoots() { JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Scan From Roots", startTicks); } } finally { - JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Snapshot Heap", startTicks); + scanFromRootsTimer.stop(); } + } - startTicks = JfrGCEvents.startGCPhasePause(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void scanFromDirtyRoots() { + Timer scanFromDirtyRootsTimer = timers.scanFromDirtyRoots.start(); try { - /* - * Make sure all aligned chunks with pinned objects are in To spaces so that pinned - * objects stay alive and cannot move. - */ - promoteChunksWithPinnedObjects(); - } finally { - JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Promote Pinned Objects", startTicks); - } + long startTicks = JfrGCEvents.startGCPhasePause(); - startTicks = JfrGCEvents.startGCPhasePause(); - try { - if (!completeCollection) { + try { + /* Snapshot the heap so that objects that are promoted afterwards can be visited. */ + beginPromotion(true); + } finally { + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Snapshot Heap", startTicks); + } + + startTicks = JfrGCEvents.startGCPhasePause(); + try { + /* + * Make sure any released objects are in toSpace (because this is an incremental + * collection). I do this before blackening any roots to make sure the chunks with + * pinned objects are moved entirely, as opposed to promoting the objects + * individually by roots. This makes the objects in those chunks grey. + */ + promoteChunksWithPinnedObjects(); + } finally { + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Promote Pinned Objects", startTicks); + } + + startTicks = JfrGCEvents.startGCPhasePause(); + try { /* * Blacken Objects that are dirty roots. There are dirty cards in ToSpace. Do this * early so I don't have to walk the cards of individually promoted objects, which @@ -761,34 +764,32 @@ private void scanFromRoots() { blackenDirtyCardRoots(); blackenStackRoots(); blackenThreadLocals(); - JNIObjectHandles.scanGlobalHandleBuckets(globalHandlesCallback); + JNIObjectHandles.scanGlobalHandleBuckets(GLOBAL_CALLBACK); blackenDirtyImageHeapRoots(); blackenMetaspace(); } finally { JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Scan Roots", startTicks); } - blackenStackRoots(); - blackenThreadLocals(); - blackenImageHeapRoots(); - blackenMetaspace(); - } finally { - JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Scan Roots", startTicks); - } - startTicks = JfrGCEvents.startGCPhasePause(); - try { - /* Visit all the Objects promoted since the snapshot. */ - scanGreyObjects(); + startTicks = JfrGCEvents.startGCPhasePause(); + try { + /* Visit all the Objects promoted since the snapshot, transitively. */ + scanGreyObjects(true); - if (RuntimeCompilation.isEnabled()) { - /* Visit the runtime compiled code, now that we know all the reachable objects. */ - walkRuntimeCodeCache(); + if (RuntimeCompilation.isEnabled()) { + /* + * Visit the runtime compiled code, now that we know all the reachable objects. + */ + walkRuntimeCodeCache(); - /* Visit all objects that became reachable because of the compiled code. */ - scanGreyObjects(); + /* Visit all objects that became reachable because of the compiled code. */ + scanGreyObjects(true); + } + } finally { + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Scan From Roots", startTicks); } } finally { - JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Scan From Roots", startTicks); + scanFromDirtyRootsTimer.stop(); } } @@ -915,31 +916,22 @@ private void blackenThreadLocals() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void blackenImageHeapRoots() { + private void blackenDirtyImageHeapRoots() { + if (!HeapImpl.usesImageHeapCardMarking()) { + blackenImageHeapRoots(); + return; + } + Timer blackenImageHeapRootsTimer = timers.blackenImageHeapRoots.start(); try { - /* - * Avoid scanning the entire image heap even for complete collections: its remembered - * set contains references into both the runtime heap's old and young generations. - */ - boolean onlyDirty = HeapImpl.usesImageHeapCardMarking(); - for (ImageHeapInfo info : HeapImpl.getImageHeapInfos()) { - if (onlyDirty) { - blackenDirtyImageHeapChunkRoots(info); - } else { - blackenImageHeapRoots(info); - } + blackenDirtyImageHeapChunkRoots(info); } if (AuxiliaryImageHeap.isPresent()) { ImageHeapInfo auxInfo = AuxiliaryImageHeap.singleton().getImageHeapInfo(); if (auxInfo != null) { - if (onlyDirty) { - blackenDirtyImageHeapChunkRoots(auxInfo); - } else { - blackenImageHeapRoots(auxInfo); - } + blackenDirtyImageHeapChunkRoots(auxInfo); } } } finally { @@ -960,10 +952,35 @@ private void blackenDirtyImageHeapChunkRoots(ImageHeapInfo info) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static void walkDirtyImageHeapChunkRoots(ImageHeapInfo info, UninterruptibleObjectVisitor visitor, UninterruptibleObjectReferenceVisitor refVisitor, boolean clean) { - assert HeapImpl.usesImageHeapCardMarking(); RememberedSet.get().walkDirtyObjects(info.getFirstWritableAlignedChunk(), info.getFirstWritableUnalignedChunk(), info.getLastWritableUnalignedChunk(), visitor, refVisitor, clean); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void blackenImageHeapRoots() { + if (HeapImpl.usesImageHeapCardMarking()) { + // Avoid scanning the entire image heap even for complete collections: its remembered + // set contains references into both the runtime heap's old and young generations. + blackenDirtyImageHeapRoots(); + return; + } + + Timer blackenImageHeapRootsTimer = timers.blackenImageHeapRoots.start(); + try { + for (ImageHeapInfo info : HeapImpl.getImageHeapInfos()) { + blackenImageHeapRoots(info); + } + + if (AuxiliaryImageHeap.isPresent()) { + ImageHeapInfo auxImageHeapInfo = AuxiliaryImageHeap.singleton().getImageHeapInfo(); + if (auxImageHeapInfo != null) { + blackenImageHeapRoots(auxImageHeapInfo); + } + } + } finally { + blackenImageHeapRootsTimer.stop(); + } + } + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).") private void blackenImageHeapRoots(ImageHeapInfo imageHeapInfo) { walkImageHeapRoots(imageHeapInfo, greyToBlackObjectVisitor); @@ -972,13 +989,12 @@ private void blackenImageHeapRoots(ImageHeapInfo imageHeapInfo) { @AlwaysInline("GC Performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static void walkImageHeapRoots(ImageHeapInfo imageHeapInfo, ObjectVisitor visitor) { - ImageHeapWalker.walkPartitionInline(imageHeapInfo.firstAlignedWritableObject, imageHeapInfo.lastAlignedWritableObject, visitor, true); - ImageHeapWalker.walkPartitionInline(imageHeapInfo.firstUnalignedWritableObject, imageHeapInfo.lastUnalignedWritableObject, visitor, false); + ImageHeapWalker.walkPartitionInline(imageHeapInfo.firstWritableRegularObject, imageHeapInfo.lastWritableRegularObject, visitor, true); + ImageHeapWalker.walkPartitionInline(imageHeapInfo.firstWritableHugeObject, imageHeapInfo.lastWritableHugeObject, visitor, false); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void blackenDirtyCardRoots() { - assert !completeCollection : "only call for incremental collections"; Timer blackenDirtyCardRootsTimer = timers.blackenDirtyCardRoots.start(); try { /* @@ -1011,22 +1027,22 @@ private void blackenMetaspace() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void beginPromotion() { + private static void beginPromotion(boolean isIncremental) { HeapImpl heap = HeapImpl.getHeapImpl(); - heap.getOldGeneration().beginPromotion(completeCollection); - if (!completeCollection) { + heap.getOldGeneration().beginPromotion(isIncremental); + if (isIncremental) { heap.getYoungGeneration().beginPromotion(); } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void scanGreyObjects() { + private void scanGreyObjects(boolean isIncremental) { Timer scanGreyObjectsTimer = timers.scanGreyObjects.start(); try { - if (completeCollection) { - HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(true); - } else { + if (isIncremental) { incrementalScanGreyObjectsLoop(); + } else { + HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(false); } } finally { scanGreyObjectsTimer.stop(); @@ -1034,15 +1050,14 @@ private void scanGreyObjects() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void incrementalScanGreyObjectsLoop() { - assert !completeCollection; + private static void incrementalScanGreyObjectsLoop() { HeapImpl heap = HeapImpl.getHeapImpl(); YoungGeneration youngGen = heap.getYoungGeneration(); OldGeneration oldGen = heap.getOldGeneration(); boolean hasGrey; do { hasGrey = youngGen.scanGreyObjects(); - hasGrey |= oldGen.scanGreyObjects(false); + hasGrey |= oldGen.scanGreyObjects(true); } while (hasGrey); }