Skip to content

Commit 16a0a50

Browse files
christianhaeublchristianhaeubl
committed
[GR-65480] Update TLAB logic to JDK 25+25 and fix gcWaste computation.
PullRequest: graal/21018
2 parents 70e5b13 + 089e691 commit 16a0a50

File tree

2 files changed

+77
-88
lines changed

2 files changed

+77
-88
lines changed

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java‎

Lines changed: 1 addition & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,6 @@
2525
package com.oracle.svm.core.genscavenge;
2626

2727
import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
28-
import static com.oracle.svm.core.genscavenge.TlabSupport.computeMinSizeOfNewTlab;
29-
import static com.oracle.svm.core.genscavenge.TlabSupport.computeSizeOfNewTlab;
30-
import static com.oracle.svm.core.genscavenge.TlabSupport.fillTlab;
31-
import static com.oracle.svm.core.genscavenge.TlabSupport.recordSlowAllocation;
32-
import static com.oracle.svm.core.genscavenge.TlabSupport.retireTlabBeforeAllocation;
33-
import static com.oracle.svm.core.genscavenge.TlabSupport.shouldRetainTlab;
3428
import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_END_IDENTITY;
3529
import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_START_IDENTITY;
3630
import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_TOP_IDENTITY;
@@ -44,7 +38,6 @@
4438
import org.graalvm.nativeimage.c.struct.RawStructure;
4539
import org.graalvm.nativeimage.c.struct.SizeOf;
4640
import org.graalvm.nativeimage.c.struct.UniqueLocationIdentity;
47-
import org.graalvm.nativeimage.c.type.WordPointer;
4841
import org.graalvm.word.LocationIdentity;
4942
import org.graalvm.word.Pointer;
5043
import org.graalvm.word.PointerBase;
@@ -364,58 +357,13 @@ private static Object allocateArraySlow(DynamicHub hub, int length, UnsignedWord
364357
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/memAllocator.cpp#L333-L341")
365358
@Uninterruptible(reason = "Holds uninitialized memory.")
366359
private static Pointer allocateRawMemory(UnsignedWord size, BooleanPointer allocatedOutsideTlab) {
367-
Pointer memory = allocateRawMemoryInTlabSlow(size);
360+
Pointer memory = TlabSupport.allocateRawMemoryInTlabSlow(size);
368361
if (memory.isNonNull()) {
369362
return memory;
370363
}
371364
return allocateRawMemoryOutsideTlab(size, allocatedOutsideTlab);
372365
}
373366

374-
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/memAllocator.cpp#L257-L329")
375-
@Uninterruptible(reason = "Holds uninitialized memory.")
376-
private static Pointer allocateRawMemoryInTlabSlow(UnsignedWord size) {
377-
ThreadLocalAllocation.Descriptor tlab = getTlab();
378-
379-
/*
380-
* Retain tlab and allocate object as an heap allocation if the amount free in the tlab is
381-
* too large to discard.
382-
*/
383-
if (shouldRetainTlab(tlab)) {
384-
recordSlowAllocation();
385-
return Word.nullPointer();
386-
}
387-
388-
/*
389-
* Discard tlab and allocate a new one. To minimize fragmentation, the last tlab may be
390-
* smaller than the rest.
391-
*/
392-
UnsignedWord newTlabSize = computeSizeOfNewTlab(size);
393-
394-
retireTlabBeforeAllocation();
395-
396-
if (newTlabSize.equal(0)) {
397-
return Word.nullPointer();
398-
}
399-
400-
/*
401-
* Allocate a new TLAB requesting newTlabSize. Any size between minimal and newTlabSize is
402-
* accepted.
403-
*/
404-
405-
UnsignedWord computedMinSize = computeMinSizeOfNewTlab(size);
406-
407-
WordPointer allocatedTlabSize = StackValue.get(WordPointer.class);
408-
Pointer memory = YoungGeneration.getHeapAllocation().allocateNewTlab(computedMinSize, newTlabSize, allocatedTlabSize);
409-
if (memory.isNull()) {
410-
assert Word.unsigned(0).equal(allocatedTlabSize.read()) : "Allocation failed, but actual size was updated.";
411-
return Word.nullPointer();
412-
}
413-
assert Word.unsigned(0).notEqual(allocatedTlabSize.read()) : "Allocation succeeded but actual size not updated.";
414-
415-
fillTlab(memory, memory.add(size), allocatedTlabSize);
416-
return memory;
417-
}
418-
419367
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/memAllocator.cpp#L239-L251")
420368
@Uninterruptible(reason = "Holds uninitialized memory.")
421369
private static Pointer allocateRawMemoryOutsideTlab(UnsignedWord size, BooleanPointer allocatedOutsideTlab) {

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/TlabSupport.java‎

Lines changed: 76 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434

3535
import org.graalvm.nativeimage.CurrentIsolate;
3636
import org.graalvm.nativeimage.IsolateThread;
37+
import org.graalvm.nativeimage.StackValue;
3738
import org.graalvm.nativeimage.c.struct.SizeOf;
3839
import org.graalvm.nativeimage.c.type.WordPointer;
3940
import org.graalvm.word.Pointer;
@@ -70,7 +71,7 @@
7071
* and another thread allocates nothing. Between GC 23 and GC 24 the allocation behaviour of these
7172
* two threads switches. The allocation average and the TLAB size adapt to the new allocation
7273
* behavior.
73-
*
74+
*
7475
* <pre>
7576
* +-----+---------------------------------------++---------------------------------------+
7677
* | #GC | Thread 1 || Thread 2 |
@@ -87,7 +88,7 @@
8788
* | 29 | 0B | 270,44kB | 5,41kB || 3,55MB | 3,28MB | 67,14kB |
8889
* +-----+--------------+------------+-----------++--------------+------------+-----------+
8990
* </pre>
90-
*
91+
* <p>
9192
* A thread allocating a very large amount of memory will also have a high
9293
* {@link #allocatedBytesAvg}. If such a thread later changes its allocation behaviour and only
9394
* allocates a small amount of memory the {@link #allocatedBytesAvg} starts decreasing with the next
@@ -109,25 +110,23 @@ public class TlabSupport {
109110
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/tlab_globals.hpp#L82-L85")//
110111
private static final long TLAB_WASTE_INCREMENT = 4;
111112

112-
// The desired size of the TLAB, including the reserve for filling the unused memory.
113+
/* The desired size of the TLAB, including the reserve for filling the unused memory. */
113114
private static final FastThreadLocalWord<UnsignedWord> desiredSize = FastThreadLocalFactory.createWord("TlabSupport.desiredSize");
114-
115115
private static final FastThreadLocalWord<UnsignedWord> tlabAllocatedAlignedBytesBeforeLastGC = FastThreadLocalFactory.createWord("TlabSupport.tlabAllocatedAlignedBytesBeforeLastGC");
116-
117116
private static final FastThreadLocalInt numberOfRefills = FastThreadLocalFactory.createInt("TlabSupport.numberOfRefills");
118117
private static final FastThreadLocalInt refillWaste = FastThreadLocalFactory.createInt("TlabSupport.refillWaste");
119118
private static final FastThreadLocalInt gcWaste = FastThreadLocalFactory.createInt("TlabSupport.gcWaste");
120119

121-
// Average of allocated bytes in TLABs of this thread.
120+
/* Average of allocated bytes in TLABs of this thread. */
122121
private static final FastThreadLocalBytes<AdaptiveWeightedAverageStruct.Data> allocatedBytesAvg = FastThreadLocalFactory
123122
.createBytes(() -> SizeOf.get(AdaptiveWeightedAverageStruct.Data.class), "TlabSupport.allocatedBytesAvg");
124123

125-
// Hold onto the TLAB if availableTlabMemory() is larger than this.
124+
/* Hold onto the TLAB if availableTlabMemory() is larger than this. */
126125
private static final FastThreadLocalWord<UnsignedWord> refillWasteLimit = FastThreadLocalFactory.createWord("TlabSupport.refillWasteLimit");
127126

128127
private static final FastThreadLocalInt slowAllocations = FastThreadLocalFactory.createInt("TlabSupport.slowAllocations");
129128

130-
// Expected number of refills between GCs.
129+
/* Expected number of refills between GCs. */
131130
private static UnsignedWord targetRefills = Word.unsigned(1);
132131

133132
private static boolean initialized;
@@ -160,9 +159,53 @@ public static void initialize(IsolateThread thread) {
160159
resetStatistics(thread);
161160
}
162161

162+
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/memAllocator.cpp#L257-L329")
163+
@Uninterruptible(reason = "Holds uninitialized memory.")
164+
static Pointer allocateRawMemoryInTlabSlow(UnsignedWord size) {
165+
ThreadLocalAllocation.Descriptor tlab = getTlab();
166+
167+
/*
168+
* Retain tlab and allocate object as an heap allocation if the amount free in the tlab is
169+
* too large to discard.
170+
*/
171+
if (shouldRetainTlab(tlab)) {
172+
recordSlowAllocation();
173+
return Word.nullPointer();
174+
}
175+
176+
/* Discard tlab and allocate a new one. */
177+
recordRefillWaste();
178+
retireTlab(CurrentIsolate.getCurrentThread(), false);
179+
180+
/* To minimize fragmentation, the last tlab may be smaller than the rest. */
181+
UnsignedWord newTlabSize = computeSizeOfNewTlab(size);
182+
if (newTlabSize.equal(0)) {
183+
return Word.nullPointer();
184+
}
185+
186+
/*
187+
* Allocate a new TLAB requesting newTlabSize. Any size between minimal and newTlabSize is
188+
* accepted.
189+
*/
190+
UnsignedWord computedMinSize = computeMinSizeOfNewTlab(size);
191+
192+
WordPointer allocatedTlabSize = StackValue.get(WordPointer.class);
193+
Pointer memory = YoungGeneration.getHeapAllocation().allocateNewTlab(computedMinSize, newTlabSize, allocatedTlabSize);
194+
if (memory.isNull()) {
195+
assert Word.unsigned(0).equal(allocatedTlabSize.read()) : "Allocation failed, but actual size was updated.";
196+
return Word.nullPointer();
197+
}
198+
assert Word.unsigned(0).notEqual(allocatedTlabSize.read()) : "Allocation succeeded but actual size not updated.";
199+
200+
fillTlab(memory, memory.add(size), allocatedTlabSize);
201+
return memory;
202+
}
203+
204+
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/runtime/thread.cpp#L168-L174")
163205
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L183-L195")
164206
@Uninterruptible(reason = "Accesses TLAB")
165-
static void fillTlab(Pointer start, Pointer top, WordPointer newSize) {
207+
private static void fillTlab(Pointer start, Pointer top, WordPointer newSize) {
208+
/* Fill the TLAB. */
166209
numberOfRefills.set(numberOfRefills.get() + 1);
167210

168211
Pointer hardEnd = start.add(newSize.read());
@@ -172,39 +215,38 @@ static void fillTlab(Pointer start, Pointer top, WordPointer newSize) {
172215

173216
initialize(getTlab(), start, top, end);
174217

175-
// Reset amount of internal fragmentation
218+
/* Reset amount of internal fragmentation. */
176219
refillWasteLimit.set(initialRefillWasteLimit());
177220
}
178221

179222
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L143-L145")
180223
@Uninterruptible(reason = "Accesses TLAB")
181-
static void retireTlabBeforeAllocation() {
224+
private static void recordRefillWaste() {
182225
long availableTlabMemory = availableTlabMemory(getTlab()).rawValue();
183226
refillWaste.set(refillWaste.get() + UninterruptibleUtils.NumUtil.safeToInt(availableTlabMemory));
184-
retireCurrentTlab(CurrentIsolate.getCurrentThread(), false);
185227
}
186228

229+
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/runtime/thread.cpp#L157-L166")
187230
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L131-L141")
188231
@Uninterruptible(reason = "Accesses TLAB")
189-
private static void retireCurrentTlab(IsolateThread thread, boolean calculateStats) {
232+
private static void retireTlab(IsolateThread thread, boolean calculateStats) {
233+
/* Sampling and serviceability support. */
190234
ThreadLocalAllocation.Descriptor tlab = getTlab(thread);
191-
192235
if (tlab.getAllocationEnd(TLAB_END_IDENTITY).isNonNull()) {
193-
assert checkInvariants(tlab);
194-
195-
UnsignedWord usedTlabSize = getUsedTlabSize(tlab);
196-
allocatedAlignedBytes.set(thread, allocatedAlignedBytes.get(thread).add(usedTlabSize));
197-
insertFiller(tlab);
198-
initialize(tlab, Word.nullPointer(), Word.nullPointer(), Word.nullPointer());
236+
UnsignedWord usedBytes = getUsedTlabSize(tlab);
237+
allocatedAlignedBytes.set(thread, allocatedAlignedBytes.get(thread).add(usedBytes));
199238
}
200239

201-
/*
202-
* Collect statistics after the TLAB has been retired. Otherwise, the current TLAB is
203-
* excluded from the statistics.
204-
*/
240+
/* Retire the TLAB. */
205241
if (calculateStats) {
206242
accumulateAndResetStatistics(thread);
207243
}
244+
245+
if (tlab.getAllocationEnd(TLAB_END_IDENTITY).isNonNull()) {
246+
assert checkInvariants(tlab);
247+
insertFiller(tlab);
248+
initialize(tlab, Word.nullPointer(), Word.nullPointer(), Word.nullPointer());
249+
}
208250
}
209251

210252
@Uninterruptible(reason = "Accesses TLAB")
@@ -238,7 +280,7 @@ private static boolean checkInvariants(Descriptor tlab) {
238280
@Uninterruptible(reason = "Accesses TLAB")
239281
static void suspendAllocationInCurrentThread() {
240282
/* The statistics for this thread will be updated later. */
241-
retireCurrentTlab(CurrentIsolate.getCurrentThread(), false);
283+
retireTlab(CurrentIsolate.getCurrentThread(), false);
242284
}
243285

244286
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
@@ -268,7 +310,7 @@ static void disableAndFlushForThread(IsolateThread vmThread) {
268310
private static void retireTlabToEden(IsolateThread thread) {
269311
VMThreads.guaranteeOwnsThreadMutex("Otherwise, we wouldn't be allowed to access the space.", true);
270312

271-
retireCurrentTlab(thread, true);
313+
retireTlab(thread, true);
272314

273315
Descriptor tlab = getTlab(thread);
274316
UnalignedHeapChunk.UnalignedHeader unalignedChunk = tlab.getUnalignedChunk();
@@ -285,7 +327,7 @@ private static void retireTlabToEden(IsolateThread thread) {
285327
}
286328

287329
@Uninterruptible(reason = "Accesses TLAB")
288-
static UnsignedWord availableTlabMemory(Descriptor tlab) {
330+
private static UnsignedWord availableTlabMemory(Descriptor tlab) {
289331
Pointer top = tlab.getAllocationTop(TLAB_TOP_IDENTITY);
290332
Pointer end = tlab.getAllocationEnd(TLAB_END_IDENTITY);
291333
assert top.belowOrEqual(end);
@@ -323,7 +365,6 @@ private static void insertFiller(ThreadLocalAllocation.Descriptor tlab) {
323365
if (top.belowThan(hardEnd)) {
324366
FillerObjectUtil.writeFillerObjectAt(top, size);
325367
}
326-
327368
}
328369

329370
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L175-L181")
@@ -385,7 +426,7 @@ private static UnsignedWord initialRefillWasteLimit() {
385426

386427
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+8/src/hotspot/share/gc/shared/threadLocalAllocBuffer.inline.hpp#L54-L71")
387428
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
388-
static UnsignedWord computeSizeOfNewTlab(UnsignedWord allocationSize) {
429+
private static UnsignedWord computeSizeOfNewTlab(UnsignedWord allocationSize) {
389430
assert UnsignedUtils.isAMultiple(allocationSize, Word.unsigned(ConfigurationValues.getObjectLayout().getAlignment()));
390431

391432
/*
@@ -404,7 +445,7 @@ static UnsignedWord computeSizeOfNewTlab(UnsignedWord allocationSize) {
404445

405446
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/threadLocalAllocBuffer.inline.hpp#L73-L77")
406447
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
407-
static UnsignedWord computeMinSizeOfNewTlab(UnsignedWord allocationSize) {
448+
private static UnsignedWord computeMinSizeOfNewTlab(UnsignedWord allocationSize) {
408449
UnsignedWord alignedSize = Word.unsigned(ConfigurationValues.getObjectLayout().alignUp(allocationSize.rawValue()));
409450
UnsignedWord sizeWithReserve = alignedSize.add(getFillerObjectSize());
410451
long minTlabSize = TlabOptionCache.singleton().getMinTlabSize();
@@ -413,13 +454,13 @@ static UnsignedWord computeMinSizeOfNewTlab(UnsignedWord allocationSize) {
413454
}
414455

415456
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
416-
static boolean shouldRetainTlab(Descriptor tlab) {
457+
private static boolean shouldRetainTlab(Descriptor tlab) {
417458
return availableTlabMemory(tlab).aboveThan(refillWasteLimit.get());
418459
}
419460

420461
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+11/src/hotspot/share/gc/shared/threadLocalAllocBuffer.inline.hpp#L79-L94")
421462
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
422-
static void recordSlowAllocation() {
463+
private static void recordSlowAllocation() {
423464
/*
424465
* Raise size required to bypass TLAB next time. Else there's a risk that a thread that
425466
* repeatedly allocates objects of one size will get stuck on this slow path.
@@ -436,15 +477,16 @@ static UnsignedWord maxSize() {
436477
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L76-L117")
437478
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
438479
private static void accumulateAndResetStatistics(IsolateThread thread) {
439-
gcWaste.set(thread, gcWaste.get() + UninterruptibleUtils.NumUtil.safeToInt(availableTlabMemory(getTlab(thread)).rawValue()));
480+
UnsignedWord remaining = availableTlabMemory(getTlab());
481+
gcWaste.set(thread, gcWaste.get() + UnsignedUtils.safeToInt(remaining));
482+
440483
UnsignedWord totalAlignedAllocated = ThreadLocalAllocation.getAlignedAllocatedBytes(thread);
441484
UnsignedWord allocatedAlignedSinceLastGC = totalAlignedAllocated.subtract(tlabAllocatedAlignedBytesBeforeLastGC.get(thread));
442485
tlabAllocatedAlignedBytesBeforeLastGC.set(thread, totalAlignedAllocated);
443486

444487
AdaptiveWeightedAverageStruct.sample(allocatedBytesAvg.getAddress(thread), allocatedAlignedSinceLastGC.rawValue());
445488

446489
printStats(thread, allocatedAlignedSinceLastGC);
447-
448490
resetStatistics(thread);
449491
}
450492

@@ -469,7 +511,6 @@ static void logTlabChunks(Log log, IsolateThread thread, String shortSpaceName)
469511
ThreadLocalAllocation.Descriptor tlab = getTlabUnsafe(thread);
470512

471513
// Aligned chunks are handled in HeapAllocation.
472-
473514
UnalignedHeapChunk.UnalignedHeader uChunk = tlab.getUnalignedChunk();
474515
HeapChunkLogging.logChunks(log, uChunk, shortSpaceName, false);
475516
}

0 commit comments

Comments
 (0)