diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 00da84e47c56..4925ebad1976 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -31,6 +31,9 @@ import java.lang.ref.Reference; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.handles.ObjectHandlesImpl; +import com.oracle.svm.core.jni.JNIObjectHandles; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -39,6 +42,7 @@ import org.graalvm.nativeimage.c.struct.RawField; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; @@ -86,7 +90,6 @@ import com.oracle.svm.core.heap.UninterruptibleObjectReferenceVisitor; import com.oracle.svm.core.heap.UninterruptibleObjectVisitor; import com.oracle.svm.core.heap.VMOperationInfos; -import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; import com.oracle.svm.core.interpreter.InterpreterSupport; import com.oracle.svm.core.jdk.RuntimeSupport; import com.oracle.svm.core.jfr.JfrGCWhen; @@ -108,22 +111,15 @@ import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.VMThreadLocalSupport; -import com.oracle.svm.core.traits.BuiltinTraits.AllAccess; -import com.oracle.svm.core.traits.BuiltinTraits.NoLayeredCallbacks; -import com.oracle.svm.core.traits.BuiltinTraits.PartiallyLayerAware; -import com.oracle.svm.core.traits.SingletonLayeredInstallationKind.Independent; -import com.oracle.svm.core.traits.SingletonTraits; import com.oracle.svm.core.util.TimeUtils; import com.oracle.svm.core.util.Timer; import com.oracle.svm.core.util.VMError; - import jdk.graal.compiler.api.replacements.Fold; import jdk.graal.compiler.word.Word; /** * Garbage collector (incremental or complete) for {@link HeapImpl}. */ -@SingletonTraits(access = AllAccess.class, layeredCallbacks = NoLayeredCallbacks.class, layeredInstallationKind = Independent.class, other = PartiallyLayerAware.class) public final class GCImpl implements GC { private static final long K = 1024; static final long M = K * K; @@ -141,16 +137,15 @@ public final class GCImpl implements GC { private final CollectionPolicy policy; private boolean completeCollection = false; - private boolean outOfMemoryCollection = false; private UnsignedWord collectionEpoch = Word.zero(); private long lastWholeHeapExaminedNanos = -1; + private static final int WORD_SIZE = ConfigurationValues.getTarget().wordSize; + private final GlobalHandlesBucketCallback GLOBAL_CALLBACK = new GlobalHandlesBucketCallback(greyToBlackObjRefVisitor); @Platforms(Platform.HOSTED_ONLY.class) GCImpl() { this.policy = CollectionPolicy.getInitialPolicy(); - if (ImageLayerBuildingSupport.firstImageBuild()) { - RuntimeSupport.getRuntimeSupport().addShutdownHook(_ -> printGCSummary()); - } + RuntimeSupport.getRuntimeSupport().addShutdownHook(_ -> printGCSummary()); } @Override @@ -309,12 +304,13 @@ private boolean collectImpl(GCCause cause, long beginNanoTime, boolean forceFull try { outOfMemory = doCollectImpl(cause, beginNanoTime, forceFullGC, false); if (outOfMemory) { - outOfMemoryCollection = true; // increase eagerness to free memory + // Avoid running out of memory with a full GC that reclaims softly reachable objects + ReferenceObjectProcessing.setSoftReferencesAreWeak(true); try { verifyHeap(During); outOfMemory = doCollectImpl(cause, System.nanoTime(), true, true); } finally { - outOfMemoryCollection = false; + ReferenceObjectProcessing.setSoftReferencesAreWeak(false); } } } finally { @@ -328,7 +324,7 @@ private boolean doCollectImpl(GCCause cause, long initialBeginNanoTime, boolean ChunkBasedCommittedMemoryProvider.get().beforeGarbageCollection(); - boolean incremental = !forceNoIncremental && !policy.shouldCollectCompletely(false, forceFullGC); + boolean incremental = !forceNoIncremental && !policy.shouldCollectCompletely(false); boolean outOfMemory = false; if (incremental) { @@ -339,7 +335,7 @@ private boolean doCollectImpl(GCCause cause, long initialBeginNanoTime, boolean JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Incremental GC", startTicks); } } - if (!incremental || outOfMemory || forceFullGC || policy.shouldCollectCompletely(incremental, forceFullGC)) { + if (!incremental || outOfMemory || forceFullGC || policy.shouldCollectCompletely(incremental)) { long beginNanoTime = initialBeginNanoTime; if (incremental) { beginNanoTime = System.nanoTime(); @@ -369,7 +365,7 @@ private boolean doCollectOnce(GCCause cause, long beginNanoTime, boolean complet accounting.beforeCollectOnce(completeCollection); policy.onCollectionBegin(completeCollection, beginNanoTime); - doCollectCore(); + doCollectCore(!complete); if (complete) { lastWholeHeapExaminedNanos = System.nanoTime(); } @@ -434,7 +430,7 @@ public static UnsignedWord getChunkBytes() { } private static void resizeAllTlabs() { - if (SubstrateGCOptions.ResizeTLAB.getValue()) { + if (SubstrateGCOptions.TlabOptions.ResizeTLAB.getValue()) { for (IsolateThread thread = VMThreads.firstThread(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) { TlabSupport.resize(thread); } @@ -526,24 +522,13 @@ public void collectCompletely(GCCause cause) { collect(cause, true); } - @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isCompleteCollection() { return completeCollection; } - /** - * Whether the current collection is intended to be more aggressive as a last resort to avoid an - * out of memory condition. - */ - @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public boolean isOutOfMemoryCollection() { - return outOfMemoryCollection; - } - /** Collect, either incrementally or completely, and process discovered references. */ - private void doCollectCore() { + private void doCollectCore(boolean incremental) { GreyToBlackObjRefVisitor.Counters counters = greyToBlackObjRefVisitor.openCounters(); long startTicks; try { @@ -552,15 +537,15 @@ private void doCollectCore() { startTicks = JfrGCEvents.startGCPhasePause(); try { /* Scan reachable objects and potentially already copy them once discovered. */ - scan(); + scan(incremental); } finally { - JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), completeCollection ? "Scan" : "Incremental Scan", startTicks); + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), incremental ? "Incremental Scan" : "Scan", startTicks); } } finally { rootScanTimer.stop(); } - if (completeCollection) { + if (!incremental) { /* Sweep or compact objects in the old generation unless already done by copying. */ HeapImpl.getHeapImpl().getOldGeneration().sweepAndCompact(timers, chunkReleaser); } @@ -610,7 +595,7 @@ private void doCollectCore() { * chunks for copying live old objects with fewer chunk allocations. In either * case, excess chunks are released later. */ - boolean keepAllAlignedChunks = !SerialGCOptions.useCompactingOldGen() && !completeCollection; + boolean keepAllAlignedChunks = !SerialGCOptions.useCompactingOldGen() && incremental; chunkReleaser.release(keepAllAlignedChunks); } finally { JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Release Spaces", startTicks); @@ -655,70 +640,156 @@ private void cleanRuntimeCodeCache() { } @Uninterruptible(reason = "We don't want any safepoint checks in the core part of the GC.") - private void scan() { - Timer timer = completeCollection ? timers.scanFromRoots : timers.scanFromDirtyRoots; - timer.start(); - try { + private void scan(boolean incremental) { + if (incremental) { + scanFromDirtyRoots(); + } else { scanFromRoots(); - } finally { - timer.stop(); } } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void scanFromRoots() { - long startTicks = JfrGCEvents.startGCPhasePause(); + static final class GlobalHandlesBucketCallback implements ObjectHandlesImpl.BucketCallback { + GreyToBlackObjRefVisitor v; - try { - /* Snapshot the heap so that objects that are promoted afterwards can be visited. */ - beginPromotion(); - } finally { - JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Snapshot Heap", startTicks); + private GlobalHandlesBucketCallback(GreyToBlackObjRefVisitor v) { + this.v = v; } - startTicks = JfrGCEvents.startGCPhasePause(); + @Override + @Uninterruptible(reason = "Construct callback with visitor") + public void visitBucket(Pointer bucketAddress, int count) { + v.visitObjectReferences( + bucketAddress, + false, + WORD_SIZE, + null, + count + ); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void scanFromRoots() { + Timer scanFromRootsTimer = timers.scanFromRoots.start(); try { - /* - * Make sure all aligned chunks with pinned objects are in To spaces so that pinned - * objects stay alive and cannot move. - */ - promoteChunksWithPinnedObjects(); + long startTicks = JfrGCEvents.startGCPhasePause(); + try { + /* + * Snapshot the heap so that objects that are promoted afterwards can be visited. + * When using a compacting old generation, it absorbs all chunks from the young + * generation at this point. + */ + beginPromotion(false); + } finally { + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Snapshot Heap", startTicks); + } + + startTicks = JfrGCEvents.startGCPhasePause(); + try { + /* + * Make sure all chunks with pinned objects are in toSpace, and any formerly pinned + * objects are in fromSpace. + */ + promoteChunksWithPinnedObjects(); + } finally { + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Promote Pinned Objects", startTicks); + } + + startTicks = JfrGCEvents.startGCPhasePause(); + try { + blackenStackRoots(); + blackenThreadLocals(); + JNIObjectHandles.scanGlobalHandleBuckets(GLOBAL_CALLBACK); + blackenImageHeapRoots(); + blackenMetaspace(); + } finally { + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Scan Roots", startTicks); + } + + startTicks = JfrGCEvents.startGCPhasePause(); + try { + /* Visit all the Objects promoted since the snapshot. */ + scanGreyObjects(false); + + if (RuntimeCompilation.isEnabled()) { + /* + * Visit the runtime compiled code, now that we know all the reachable objects. + */ + walkRuntimeCodeCache(); + + /* Visit all objects that became reachable because of the compiled code. */ + scanGreyObjects(false); + } + } finally { + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Scan From Roots", startTicks); + } } finally { - JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Promote Pinned Objects", startTicks); + scanFromRootsTimer.stop(); } + } - startTicks = JfrGCEvents.startGCPhasePause(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void scanFromDirtyRoots() { + Timer scanFromDirtyRootsTimer = timers.scanFromDirtyRoots.start(); try { - if (!completeCollection) { + long startTicks = JfrGCEvents.startGCPhasePause(); + + try { + /* Snapshot the heap so that objects that are promoted afterwards can be visited. */ + beginPromotion(true); + } finally { + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Snapshot Heap", startTicks); + } + + startTicks = JfrGCEvents.startGCPhasePause(); + try { + /* + * Make sure any released objects are in toSpace (because this is an incremental + * collection). I do this before blackening any roots to make sure the chunks with + * pinned objects are moved entirely, as opposed to promoting the objects + * individually by roots. This makes the objects in those chunks grey. + */ + promoteChunksWithPinnedObjects(); + } finally { + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Promote Pinned Objects", startTicks); + } + + startTicks = JfrGCEvents.startGCPhasePause(); + try { /* * Blacken Objects that are dirty roots. There are dirty cards in ToSpace. Do this * early so I don't have to walk the cards of individually promoted objects, which * will be visited by the grey object scanner. */ blackenDirtyCardRoots(); + blackenStackRoots(); + blackenThreadLocals(); + JNIObjectHandles.scanGlobalHandleBuckets(GLOBAL_CALLBACK); + blackenDirtyImageHeapRoots(); + blackenMetaspace(); + } finally { + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Scan Roots", startTicks); } - blackenStackRoots(); - blackenThreadLocals(); - blackenImageHeapRoots(); - blackenMetaspace(); - } finally { - JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Scan Roots", startTicks); - } - startTicks = JfrGCEvents.startGCPhasePause(); - try { - /* Visit all the Objects promoted since the snapshot. */ - scanGreyObjects(); + startTicks = JfrGCEvents.startGCPhasePause(); + try { + /* Visit all the Objects promoted since the snapshot, transitively. */ + scanGreyObjects(true); - if (RuntimeCompilation.isEnabled()) { - /* Visit the runtime compiled code, now that we know all the reachable objects. */ - walkRuntimeCodeCache(); + if (RuntimeCompilation.isEnabled()) { + /* + * Visit the runtime compiled code, now that we know all the reachable objects. + */ + walkRuntimeCodeCache(); - /* Visit all objects that became reachable because of the compiled code. */ - scanGreyObjects(); + /* Visit all objects that became reachable because of the compiled code. */ + scanGreyObjects(true); + } + } finally { + JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Scan From Roots", startTicks); } } finally { - JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), "Scan From Roots", startTicks); + scanFromDirtyRootsTimer.stop(); } } @@ -845,31 +916,22 @@ private void blackenThreadLocals() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void blackenImageHeapRoots() { + private void blackenDirtyImageHeapRoots() { + if (!HeapImpl.usesImageHeapCardMarking()) { + blackenImageHeapRoots(); + return; + } + Timer blackenImageHeapRootsTimer = timers.blackenImageHeapRoots.start(); try { - /* - * Avoid scanning the entire image heap even for complete collections: its remembered - * set contains references into both the runtime heap's old and young generations. - */ - boolean onlyDirty = HeapImpl.usesImageHeapCardMarking(); - for (ImageHeapInfo info : HeapImpl.getImageHeapInfos()) { - if (onlyDirty) { - blackenDirtyImageHeapChunkRoots(info); - } else { - blackenImageHeapRoots(info); - } + blackenDirtyImageHeapChunkRoots(info); } if (AuxiliaryImageHeap.isPresent()) { ImageHeapInfo auxInfo = AuxiliaryImageHeap.singleton().getImageHeapInfo(); if (auxInfo != null) { - if (onlyDirty) { - blackenDirtyImageHeapChunkRoots(auxInfo); - } else { - blackenImageHeapRoots(auxInfo); - } + blackenDirtyImageHeapChunkRoots(auxInfo); } } } finally { @@ -890,10 +952,35 @@ private void blackenDirtyImageHeapChunkRoots(ImageHeapInfo info) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static void walkDirtyImageHeapChunkRoots(ImageHeapInfo info, UninterruptibleObjectVisitor visitor, UninterruptibleObjectReferenceVisitor refVisitor, boolean clean) { - assert HeapImpl.usesImageHeapCardMarking(); RememberedSet.get().walkDirtyObjects(info.getFirstWritableAlignedChunk(), info.getFirstWritableUnalignedChunk(), info.getLastWritableUnalignedChunk(), visitor, refVisitor, clean); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void blackenImageHeapRoots() { + if (HeapImpl.usesImageHeapCardMarking()) { + // Avoid scanning the entire image heap even for complete collections: its remembered + // set contains references into both the runtime heap's old and young generations. + blackenDirtyImageHeapRoots(); + return; + } + + Timer blackenImageHeapRootsTimer = timers.blackenImageHeapRoots.start(); + try { + for (ImageHeapInfo info : HeapImpl.getImageHeapInfos()) { + blackenImageHeapRoots(info); + } + + if (AuxiliaryImageHeap.isPresent()) { + ImageHeapInfo auxImageHeapInfo = AuxiliaryImageHeap.singleton().getImageHeapInfo(); + if (auxImageHeapInfo != null) { + blackenImageHeapRoots(auxImageHeapInfo); + } + } + } finally { + blackenImageHeapRootsTimer.stop(); + } + } + @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).") private void blackenImageHeapRoots(ImageHeapInfo imageHeapInfo) { walkImageHeapRoots(imageHeapInfo, greyToBlackObjectVisitor); @@ -902,13 +989,12 @@ private void blackenImageHeapRoots(ImageHeapInfo imageHeapInfo) { @AlwaysInline("GC Performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static void walkImageHeapRoots(ImageHeapInfo imageHeapInfo, ObjectVisitor visitor) { - ImageHeapWalker.walkPartitionInline(imageHeapInfo.firstAlignedWritableObject, imageHeapInfo.lastAlignedWritableObject, visitor, true); - ImageHeapWalker.walkPartitionInline(imageHeapInfo.firstUnalignedWritableObject, imageHeapInfo.lastUnalignedWritableObject, visitor, false); + ImageHeapWalker.walkPartitionInline(imageHeapInfo.firstWritableRegularObject, imageHeapInfo.lastWritableRegularObject, visitor, true); + ImageHeapWalker.walkPartitionInline(imageHeapInfo.firstWritableHugeObject, imageHeapInfo.lastWritableHugeObject, visitor, false); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void blackenDirtyCardRoots() { - assert !completeCollection : "only call for incremental collections"; Timer blackenDirtyCardRootsTimer = timers.blackenDirtyCardRoots.start(); try { /* @@ -941,22 +1027,22 @@ private void blackenMetaspace() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void beginPromotion() { + private static void beginPromotion(boolean isIncremental) { HeapImpl heap = HeapImpl.getHeapImpl(); - heap.getOldGeneration().beginPromotion(completeCollection); - if (!completeCollection) { + heap.getOldGeneration().beginPromotion(isIncremental); + if (isIncremental) { heap.getYoungGeneration().beginPromotion(); } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void scanGreyObjects() { + private void scanGreyObjects(boolean isIncremental) { Timer scanGreyObjectsTimer = timers.scanGreyObjects.start(); try { - if (completeCollection) { - HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(true); - } else { + if (isIncremental) { incrementalScanGreyObjectsLoop(); + } else { + HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(false); } } finally { scanGreyObjectsTimer.stop(); @@ -964,15 +1050,14 @@ private void scanGreyObjects() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void incrementalScanGreyObjectsLoop() { - assert !completeCollection; + private static void incrementalScanGreyObjectsLoop() { HeapImpl heap = HeapImpl.getHeapImpl(); YoungGeneration youngGen = heap.getYoungGeneration(); OldGeneration oldGen = heap.getOldGeneration(); boolean hasGrey; do { hasGrey = youngGen.scanGreyObjects(); - hasGrey |= oldGen.scanGreyObjects(false); + hasGrey |= oldGen.scanGreyObjects(true); } while (hasGrey); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java index 6c5d6417dd45..2d73327746b4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ObjectHandlesImpl.java @@ -24,24 +24,24 @@ */ package com.oracle.svm.core.handles; -import java.lang.ref.WeakReference; - -import jdk.graal.compiler.word.Word; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import org.graalvm.nativeimage.ObjectHandle; import org.graalvm.nativeimage.ObjectHandles; +import org.graalvm.word.LocationIdentity; +import org.graalvm.word.Pointer; import org.graalvm.word.SignedWord; import org.graalvm.word.WordBase; - -import jdk.internal.misc.Unsafe; +import com.oracle.svm.core.config.ConfigurationValues; +import jdk.graal.compiler.word.Word; +import org.graalvm.word.WordFactory; /** * This class implements {@link ObjectHandle word}-sized integer handles that refer to Java objects. * {@link #create(Object) Creating}, {@link #get(ObjectHandle) dereferencing} and * {@link #destroy(ObjectHandle) destroying} handles is thread-safe and the handles themselves are - * valid across threads. This class also supports weak handles, with which the referenced object may - * be garbage-collected, after which {@link #get(ObjectHandle)} returns {@code null}. Still, weak - * handles must also be {@link #destroyWeak(ObjectHandle) explicitly destroyed} to reclaim their - * handle value. + * valid across threads. *

* The implementation uses a variable number of object arrays, in which each array element * represents a handle. The array element's index determines the handle's integer value, and the @@ -53,14 +53,8 @@ */ public final class ObjectHandlesImpl implements ObjectHandles { - /** Private subclass to distinguish from regular handles to {@link WeakReference} objects. */ - private static final class HandleWeakReference extends WeakReference { - HandleWeakReference(T referent) { - super(referent); - } - } - private static final int MAX_FIRST_BUCKET_CAPACITY = 1024; + static { // must be a power of 2 for the arithmetic below to work assert Integer.lowestOneBit(MAX_FIRST_BUCKET_CAPACITY) == MAX_FIRST_BUCKET_CAPACITY; } @@ -69,7 +63,8 @@ private static final class HandleWeakReference extends WeakReference { private final SignedWord rangeMax; private final SignedWord nullHandle; - private final Object[][] buckets; + private final Pointer[] buckets; + private final int[] capacities; private volatile long unusedHandleSearchIndex = 0; public ObjectHandlesImpl() { @@ -86,12 +81,38 @@ public ObjectHandlesImpl(SignedWord rangeMin, SignedWord rangeMax, SignedWord nu long maxIndex = toIndex(rangeMax); int lastBucketIndex = getBucketIndex(maxIndex); int lastBucketCapacity = getIndexInBucket(maxIndex) + 1; - buckets = new Object[lastBucketIndex + 1][]; + buckets = new Pointer[lastBucketIndex + 1]; + capacities = new int[lastBucketIndex + 1]; int firstBucketCapacity = MAX_FIRST_BUCKET_CAPACITY; if (lastBucketIndex == 0) { // if our range is small, we may have only a single small bucket firstBucketCapacity = lastBucketCapacity; } - buckets[0] = new Object[firstBucketCapacity]; + capacities[0] = firstBucketCapacity; + + Pointer nullPtr = WordFactory.nullPointer(); + for (int i = 0; i < buckets.length; i++) { + buckets[i] = nullPtr; + } + } + + public interface BucketCallback { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void visitBucket(Pointer bucketAddress, int capacity); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void scanAllBuckets(BucketCallback callback) { + for (int i = 0; i < buckets.length; i++) { + Pointer bucket = getBucket(i); + if (bucket.isNull()) { + continue; + } + int cap = capacities[i]; + if (cap == 0) { + continue; + } + callback.visitBucket(bucket, cap); + } } public boolean isInRange(ObjectHandle handle) { @@ -124,19 +145,33 @@ private static int getIndexInBucket(long index) { return Math.toIntExact(bucketBit ^ displacedIndex); } - private static long getObjectArrayByteOffset(int index) { - return Unsafe.getUnsafe().arrayBaseOffset(Object[].class) + index * Unsafe.getUnsafe().arrayIndexScale(Object[].class); - } - - private Object[] getBucket(int bucketIndex) { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Pointer getBucket(int bucketIndex) { // buckets[i] is changed only once from null to its final value: try without volatile first - Object[] bucket = buckets[bucketIndex]; - if (bucket == null) { - bucket = (Object[]) Unsafe.getUnsafe().getReferenceVolatile(buckets, getObjectArrayByteOffset(bucketIndex)); - } + Pointer bucket = buckets[bucketIndex]; return bucket; } + private static int wordSize() { + return ConfigurationValues.getTarget().wordSize; + } + + private static Pointer slotPointer(Pointer bucket, int indexInBucket) { + return bucket.add(indexInBucket * wordSize()); + } + + private static Object readSlot(Pointer bucket, int indexInBucket) { + return slotPointer(bucket, indexInBucket).readObject(0); + } + + private static void writeSlot(Pointer bucket, int indexInBucket, Object value) { + slotPointer(bucket, indexInBucket).writeObject(0, value); + } + + private static boolean casSlot(Pointer bucket, int indexInBucket, Object expected, Object value) { + return slotPointer(bucket, indexInBucket).logicCompareAndSwapObject(0, expected, value, LocationIdentity.ANY_LOCATION); + } + @Override public ObjectHandle create(Object obj) { /* @@ -149,6 +184,19 @@ public ObjectHandle create(Object obj) { if (obj == null) { return (ObjectHandle) nullHandle; } + + if (buckets[0].isNull()) { + synchronized (this) { + if (buckets[0].isNull()) { + Pointer p = NullableNativeMemory.calloc(capacities[0] * wordSize(), NmtCategory.JNI); + for (int i = 0; i < capacities[0]; i++) { + writeSlot(p, i, null); + } + buckets[0] = p; + } + } + } + outer: for (;;) { long startIndex = unusedHandleSearchIndex; int startBucketIndex = getBucketIndex(startIndex); @@ -157,12 +205,14 @@ public ObjectHandle create(Object obj) { int bucketIndex = startBucketIndex; int indexInBucket = startIndexInBucket; int lastExistingBucketIndex = -1; - Object[] bucket = getBucket(bucketIndex); + Pointer bucket = getBucket(bucketIndex); + for (;;) { - while (indexInBucket < bucket.length) { - if (bucket[indexInBucket] == null) { - if (Unsafe.getUnsafe().compareAndSetReference(bucket, getObjectArrayByteOffset(indexInBucket), null, obj)) { - int newSearchIndexInBucket = (indexInBucket + 1 < bucket.length) ? (indexInBucket + 1) : indexInBucket; + while (indexInBucket < capacities[bucketIndex]) { + Object current = readSlot(bucket, indexInBucket); + if (current == null) { + if (casSlot(bucket, indexInBucket, null, obj)) { + int newSearchIndexInBucket = (indexInBucket + 1 < capacities[bucketIndex]) ? (indexInBucket + 1) : indexInBucket; unusedHandleSearchIndex = toIndex(bucketIndex, newSearchIndexInBucket); // (if the next index is in another bucket, we let the next create() // figure it out) @@ -178,29 +228,40 @@ public ObjectHandle create(Object obj) { if (lastExistingBucketIndex == getBucketIndex(maxIndex)) { throw new IllegalStateException("Handle space exhausted"); } + int newBucketIndex = lastExistingBucketIndex + 1; - if (getBucket(newBucketIndex) != null) { + if (!getBucket(newBucketIndex).isNull()) { continue outer; // start over: another thread has created a new bucket } + int newBucketCapacity = (MAX_FIRST_BUCKET_CAPACITY << newBucketIndex); if (newBucketIndex == getBucketIndex(maxIndex)) { // last bucket may be smaller newBucketCapacity = getIndexInBucket(maxIndex) + 1; } - Object[] newBucket = new Object[newBucketCapacity]; - Unsafe.getUnsafe().putReferenceVolatile(newBucket, getObjectArrayByteOffset(0), obj); - if (Unsafe.getUnsafe().compareAndSetReference(buckets, getObjectArrayByteOffset(newBucketIndex), null, newBucket)) { - unusedHandleSearchIndex = toIndex(newBucketIndex, 1); - return toHandle(newBucketIndex, 0); + + Pointer newBucket = NullableNativeMemory.calloc(newBucketCapacity * wordSize(), NmtCategory.JNI); + for (int i = 0; i < newBucketCapacity; i++) { + writeSlot(newBucket, i, null); + } + writeSlot(newBucket, 0, obj); + synchronized (this) { + if (buckets[newBucketIndex].isNull()) { + buckets[newBucketIndex] = newBucket; + capacities[newBucketIndex] = newBucketCapacity; + unusedHandleSearchIndex = toIndex(newBucketIndex, 1); + return toHandle(newBucketIndex, 0); + } } - // start over: another thread has raced us to create another bucket and won + + NullableNativeMemory.free(newBucket); continue outer; } } bucketIndex++; bucket = getBucket(bucketIndex); - if (bucket == null) { + if (bucket.isNull()) { lastExistingBucketIndex = bucketIndex - 1; bucketIndex = 0; bucket = getBucket(bucketIndex); @@ -210,17 +271,10 @@ public ObjectHandle create(Object obj) { } } - public ObjectHandle createWeak(Object obj) { - return create(new HandleWeakReference<>(obj)); - } - @SuppressWarnings("unchecked") @Override public T get(ObjectHandle handle) { Object obj = doGet(handle); - if (obj instanceof HandleWeakReference) { - obj = ((HandleWeakReference) obj).get(); - } return (T) obj; } @@ -231,17 +285,18 @@ private Object doGet(ObjectHandle handle) { if (!isInRange(handle)) { throw new IllegalArgumentException("Invalid handle"); } + long index = toIndex(handle); - Object[] bucket = getBucket(getBucketIndex(index)); - if (bucket == null) { - throw new IllegalArgumentException("Invalid handle"); - } + int bucketIndex = getBucketIndex(index); int indexInBucket = getIndexInBucket(index); - return Unsafe.getUnsafe().getReferenceVolatile(bucket, getObjectArrayByteOffset(indexInBucket)); - } - public boolean isWeak(ObjectHandle handle) { - return (doGet(handle) instanceof HandleWeakReference); + Pointer bucket = getBucket(bucketIndex); + + if (bucket.isNull()) { + throw new IllegalStateException("Bucket not allocated"); + } + + return readSlot(bucket, indexInBucket); } @Override @@ -252,26 +307,27 @@ public void destroy(ObjectHandle handle) { if (!isInRange(handle)) { throw new IllegalArgumentException("Invalid handle"); } + long index = toIndex(handle); - Object[] bucket = getBucket(getBucketIndex(index)); - if (bucket == null) { - throw new IllegalArgumentException("Invalid handle"); - } + int bucketIndex = getBucketIndex(index); int indexInBucket = getIndexInBucket(index); - Unsafe.getUnsafe().putReferenceRelease(bucket, getObjectArrayByteOffset(indexInBucket), null); - } - public void destroyWeak(ObjectHandle handle) { - destroy(handle); + Pointer bucket = getBucket(bucketIndex); + if (bucket.isNull()) { + throw new IllegalStateException("Bucket not allocated"); + } + + writeSlot(bucket, indexInBucket, null); } + public long computeCurrentCount() { long count = 0; int bucketIndex = 0; - Object[] bucket = getBucket(bucketIndex); - while (bucket != null) { - for (int i = 0; i < bucket.length; i++) { - if (bucket[i] != null) { + Pointer bucket = getBucket(bucketIndex); + while (!bucket.isNull()) { + for (int i = 0; i < capacities[bucketIndex]; i++) { + if (readSlot(bucket, i) != null) { count++; } } @@ -284,9 +340,9 @@ public long computeCurrentCount() { public long computeCurrentCapacity() { long capacity = 0; int bucketIndex = 0; - Object[] bucket = getBucket(bucketIndex); - while (bucket != null) { - capacity += bucket.length; + Pointer bucket = getBucket(bucketIndex); + while (!bucket.isNull()) { + capacity += capacities[bucketIndex]; bucketIndex++; bucket = getBucket(bucketIndex); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java index 91b39f6ec6eb..3eae294680e3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIObjectHandles.java @@ -283,6 +283,11 @@ static int getLocalHandleCount() { static long computeCurrentGlobalHandleCount() { return JNIGlobalHandles.computeCurrentCount(); } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void scanGlobalHandleBuckets(ObjectHandlesImpl.BucketCallback callback) { + JNIGlobalHandles.scanStrongHandleBuckets(callback); + } } /** @@ -299,13 +304,29 @@ final class JNIGlobalHandles { assert JNIObjectHandles.nullHandle().equal(Word.zero()); } + // Define the mid-point to split the range in half + private static final SignedWord HANDLE_RANGE_SPLIT_POINT = Word.signed(1L << 30); + private static final int HANDLE_BITS_COUNT = 31; private static final SignedWord HANDLE_BITS_MASK = Word.signed((1L << HANDLE_BITS_COUNT) - 1); private static final int VALIDATION_BITS_SHIFT = HANDLE_BITS_COUNT; - private static final int VALIDATION_BITS_COUNT = 32; + private static final int VALIDATION_BITS_COUNT = 31; private static final SignedWord VALIDATION_BITS_MASK = Word.signed((1L << VALIDATION_BITS_COUNT) - 1).shiftLeft(VALIDATION_BITS_SHIFT); + private static final SignedWord WEAK_HANDLE_FLAG = Word.signed(1L << 62); private static final SignedWord MSB = Word.signed(1L << 63); - private static final ObjectHandlesImpl globalHandles = new ObjectHandlesImpl(JNIObjectHandles.nullHandle().add(1), HANDLE_BITS_MASK, JNIObjectHandles.nullHandle()); + + // Strong global handles will occupy the lower half of the global handles range + public static final SignedWord STRONG_GLOBAL_RANGE_MIN = JNIObjectHandles.nullHandle().add(1);; + public static final SignedWord STRONG_GLOBAL_RANGE_MAX = HANDLE_RANGE_SPLIT_POINT.subtract(1); + + // Weak global handles will occupy the upper half of the global handles range + public static final SignedWord WEAK_GLOBAL_RANGE_MIN = HANDLE_RANGE_SPLIT_POINT; + public static final SignedWord WEAK_GLOBAL_RANGE_MAX = HANDLE_BITS_MASK; + + private static final ObjectHandlesImpl strongGlobalHandles + = new ObjectHandlesImpl(STRONG_GLOBAL_RANGE_MIN, STRONG_GLOBAL_RANGE_MAX, JNIObjectHandles.nullHandle()); + private static final ObjectHandlesImpl weakGlobalHandles + = new ObjectHandlesImpl(WEAK_GLOBAL_RANGE_MIN, WEAK_GLOBAL_RANGE_MAX, JNIObjectHandles.nullHandle()); @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static boolean isInRange(JNIObjectHandle handle) { @@ -317,7 +338,20 @@ private static Word isolateHash() { return Word.unsigned(isolateHash); } - private static JNIObjectHandle encode(ObjectHandle handle) { + /** + * Encodes a raw {@code ObjectHandle} into a strong {@code JNIObjectHandle}. + * * A strong handle guarantees the referenced object remains alive as long as + * the handle itself exists. + * * The handle is encoded by: + * 1. Asserting the handle fits within the available bit range. + * 2. Inserting validation bits (derived from the isolate hash) for security. + * 3. Setting the Most Significant Bit (MSB, bit 63) to mark it as an encoded handle. + * 4. The WEAK_HANDLE_FLAG bit (bit 62) remains 0. + * + * @param handle The raw, unencoded handle to the Java object. + * @return The resulting strong JNI object handle with embedded metadata. + */ + private static JNIObjectHandle encodeStrong(ObjectHandle handle) { SignedWord h = (Word) handle; if (JNIObjectHandles.haveAssertions()) { assert h.and(HANDLE_BITS_MASK).equal(h) : "unencoded handle must fit in range"; @@ -330,6 +364,24 @@ private static JNIObjectHandle encode(ObjectHandle handle) { return (JNIObjectHandle) h; } + /** + * Encodes a raw {@code ObjectHandle} into a weak {@code JNIObjectHandle}. + * * A weak handle allows the referenced object to be garbage collected even + * if the handle exists. The handle will be cleared when the object dies. + * * This method calls {@link #encodeStrong(ObjectHandle)} to perform all + * common encoding steps, and then explicitly sets the {@code WEAK_HANDLE_FLAG} + * bit (bit 62) to mark the handle as weak. + * + * @param handle The raw, unencoded handle to the Java object. + * @return The resulting weak JNI object handle with embedded metadata. + */ + private static JNIObjectHandle encodeWeak(ObjectHandle handle) { + SignedWord h = (Word) encodeStrong(handle); + h = h.or(WEAK_HANDLE_FLAG); + assert isInRange((JNIObjectHandle) h); + return (JNIObjectHandle) h; + } + private static ObjectHandle decode(JNIObjectHandle handle) { assert isInRange(handle); assert ((Word) handle).and(VALIDATION_BITS_MASK).unsignedShiftRight(VALIDATION_BITS_SHIFT) @@ -338,35 +390,55 @@ private static ObjectHandle decode(JNIObjectHandle handle) { } static T getObject(JNIObjectHandle handle) { - return globalHandles.get(decode(handle)); + if (!isInRange(handle)) { + throw new IllegalArgumentException("Invalid handle"); + } + SignedWord h = (SignedWord) handle; + if (h.and(WEAK_HANDLE_FLAG).equal(Word.zero())) { + return strongGlobalHandles.get(decode(handle)); + } else { + return weakGlobalHandles.get(decode(handle)); + } } static JNIObjectRefType getHandleType(JNIObjectHandle handle) { - assert isInRange(handle); - if (globalHandles.isWeak(decode(handle))) { + SignedWord handleValue = (Word) handle; + if (!isInRange(handle)) return JNIObjectRefType.Invalid; + if ((handleValue.and(WEAK_HANDLE_FLAG).equal(Word.zero()))) { + return JNIObjectRefType.Global; + } else { return JNIObjectRefType.WeakGlobal; } - return JNIObjectRefType.Global; } static JNIObjectHandle create(Object obj) { - return encode(globalHandles.create(obj)); + return encodeStrong(strongGlobalHandles.create(obj)); } static void destroy(JNIObjectHandle handle) { - globalHandles.destroy(decode(handle)); + strongGlobalHandles.destroy(decode(handle)); } static JNIObjectHandle createWeak(Object obj) { - return encode(globalHandles.createWeak(obj)); + return encodeWeak(weakGlobalHandles.create(obj)); } static void destroyWeak(JNIObjectHandle weakRef) { - globalHandles.destroyWeak(decode(weakRef)); + weakGlobalHandles.destroy(decode(weakRef)); } public static long computeCurrentCount() { - return globalHandles.computeCurrentCount(); + return strongGlobalHandles.computeCurrentCount() + weakGlobalHandles.computeCurrentCount(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void scanStrongHandleBuckets(ObjectHandlesImpl.BucketCallback callback) { + strongGlobalHandles.scanAllBuckets(callback); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void scanWeakHandleBuckets(ObjectHandlesImpl.BucketCallback callback) { + weakGlobalHandles.scanAllBuckets(callback); } }