Skip to content

Commit ef18876

Browse files
author
Thomas Schrott
committed
Add JFR event for object allocation outside a TLAB
1 parent 4370d98 commit ef18876

15 files changed

+224
-34
lines changed

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AddressRangeCommittedMemoryProvider.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,7 @@ protected OutOfMemoryError reportAlignedChunkAllocationFailed(int error) {
351351
}
352352

353353
@Override
354+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
354355
public Pointer allocateUnalignedChunk(UnsignedWord nbytes) {
355356
WordPointer allocOut = UnsafeStackValue.get(WordPointer.class);
356357
int error = allocateInHeapAddressSpace(nbytes, getAlignmentForUnalignedChunks(), allocOut);

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
*/
2525
package com.oracle.svm.core.genscavenge;
2626

27+
import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
28+
2729
import org.graalvm.nativeimage.Platform;
2830
import org.graalvm.nativeimage.Platforms;
2931
import org.graalvm.word.Pointer;
@@ -229,6 +231,7 @@ private void freeUnusedAlignedChunksAtSafepoint(UnsignedWord count) {
229231

230232
/** Acquire an UnalignedHeapChunk from the operating system. */
231233
@SuppressWarnings("static-method")
234+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
232235
UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) {
233236
UnsignedWord chunkSize = UnalignedHeapChunk.getChunkSizeForObject(objectSize);
234237

@@ -243,6 +246,7 @@ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) {
243246
return result;
244247
}
245248

249+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
246250
public static boolean areUnalignedChunksZeroed() {
247251
return ChunkBasedCommittedMemoryProvider.get().areUnalignedChunksZeroed();
248252
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java‎

Lines changed: 39 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -52,14 +52,14 @@
5252

5353
import com.oracle.svm.core.SubstrateGCOptions;
5454
import com.oracle.svm.core.Uninterruptible;
55+
import com.oracle.svm.core.c.BooleanPointer;
5556
import com.oracle.svm.core.config.ConfigurationValues;
5657
import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader;
5758
import com.oracle.svm.core.genscavenge.graal.GenScavengeAllocationSupport;
5859
import com.oracle.svm.core.genscavenge.graal.nodes.FormatArrayNode;
5960
import com.oracle.svm.core.genscavenge.graal.nodes.FormatObjectNode;
6061
import com.oracle.svm.core.genscavenge.graal.nodes.FormatPodNode;
6162
import com.oracle.svm.core.genscavenge.graal.nodes.FormatStoredContinuationNode;
62-
import com.oracle.svm.core.graal.snippets.DeoptTester;
6363
import com.oracle.svm.core.heap.OutOfMemoryUtil;
6464
import com.oracle.svm.core.heap.Pod;
6565
import com.oracle.svm.core.heap.RestrictHeapAccess;
@@ -236,16 +236,23 @@ public static Object slowPathNewInstance(Word objectHeader) {
236236

237237
@RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate in the implementation of allocation.")
238238
private static Object slowPathNewInstanceWithoutAllocating(DynamicHub hub, UnsignedWord size) {
239-
DeoptTester.disableDeoptTesting();
239+
HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.slowPathNewInstanceWithoutAllocating", DynamicHub.toClass(hub).getName());
240+
GCImpl.getGCImpl().maybeCollectOnAllocation(size);
241+
242+
return slowPathNewInstanceWithoutAllocation0(hub, size);
243+
}
244+
245+
@Uninterruptible(reason = "Possible use of StackValue in virtual thread.")
246+
private static Object slowPathNewInstanceWithoutAllocation0(DynamicHub hub, UnsignedWord size) {
240247
long startTicks = JfrTicks.elapsedTicks();
241-
try {
242-
HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.slowPathNewInstanceWithoutAllocating", DynamicHub.toClass(hub).getName());
243-
GCImpl.getGCImpl().maybeCollectOnAllocation(size);
244248

245-
return allocateInstanceSlow(hub, size);
249+
BooleanPointer allocatedOutsideTlab = StackValue.get(BooleanPointer.class);
250+
allocatedOutsideTlab.write(false);
251+
252+
try {
253+
return allocateInstanceSlow(hub, size, allocatedOutsideTlab);
246254
} finally {
247-
JfrAllocationEvents.emit(startTicks, hub, size, getTlabSize());
248-
DeoptTester.enableDeoptTesting();
255+
JfrAllocationEvents.emit(startTicks, hub, size, getTlabSize(), allocatedOutsideTlab.read());
249256
}
250257
}
251258

@@ -267,7 +274,7 @@ public static Object slowPathNewArrayLikeObject(Word objectHeader, int length, b
267274
throw OutOfMemoryUtil.reportOutOfMemoryError(outOfMemoryError);
268275
}
269276

270-
Object result = slowPathNewArrayLikeObject0(hub, length, size, podReferenceMap);
277+
Object result = slowPathNewArrayLikeObjectWithoutAllocating(hub, length, size, podReferenceMap);
271278

272279
runSlowPathHooks();
273280
sampleSlowPathAllocation(result, size, length);
@@ -276,14 +283,22 @@ public static Object slowPathNewArrayLikeObject(Word objectHeader, int length, b
276283
}
277284

278285
@RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate in the implementation of allocation.")
279-
private static Object slowPathNewArrayLikeObject0(DynamicHub hub, int length, UnsignedWord size, byte[] podReferenceMap) {
280-
DeoptTester.disableDeoptTesting();
286+
private static Object slowPathNewArrayLikeObjectWithoutAllocating(DynamicHub hub, int length, UnsignedWord size, byte[] podReferenceMap) {
287+
HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.slowPathNewArrayLikeObjectWithoutAllocating", DynamicHub.toClass(hub).getName());
288+
GCImpl.getGCImpl().maybeCollectOnAllocation(size);
289+
290+
return slowPathNewArrayLikeObjectWithoutAllocation0(hub, length, size, podReferenceMap);
291+
}
292+
293+
@Uninterruptible(reason = "Possible use of StackValue in virtual thread.")
294+
private static Object slowPathNewArrayLikeObjectWithoutAllocation0(DynamicHub hub, int length, UnsignedWord size, byte[] podReferenceMap) {
281295
long startTicks = JfrTicks.elapsedTicks();
282296
UnsignedWord tlabSize = Word.zero();
283-
try {
284-
HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.slowPathNewArrayOrPodWithoutAllocating", DynamicHub.toClass(hub).getName());
285-
GCImpl.getGCImpl().maybeCollectOnAllocation(size);
286297

298+
BooleanPointer allocatedOutsideTlab = StackValue.get(BooleanPointer.class);
299+
allocatedOutsideTlab.write(false);
300+
301+
try {
287302
if (!GenScavengeAllocationSupport.arrayAllocatedInAlignedChunk(size)) {
288303
/*
289304
* Large arrays go into their own unaligned chunk. Only arrays and stored
@@ -305,13 +320,12 @@ private static Object slowPathNewArrayLikeObject0(DynamicHub hub, int length, Un
305320
*/
306321
Object array = allocateSmallArrayLikeObjectInCurrentTlab(hub, length, size, podReferenceMap);
307322
if (array == null) {
308-
array = allocateArraySlow(hub, length, size, podReferenceMap);
323+
array = allocateArraySlow(hub, length, size, podReferenceMap, allocatedOutsideTlab);
309324
}
310325
tlabSize = getTlabSize();
311326
return array;
312327
} finally {
313-
JfrAllocationEvents.emit(startTicks, hub, size, tlabSize);
314-
DeoptTester.enableDeoptTesting();
328+
JfrAllocationEvents.emit(startTicks, hub, size, tlabSize, allocatedOutsideTlab.read());
315329
}
316330
}
317331

@@ -326,9 +340,9 @@ private static Object allocateInstanceInCurrentTlab(DynamicHub hub, UnsignedWord
326340
}
327341

328342
@Uninterruptible(reason = "Holds uninitialized memory.")
329-
private static Object allocateInstanceSlow(DynamicHub hub, UnsignedWord size) {
343+
private static Object allocateInstanceSlow(DynamicHub hub, UnsignedWord size, BooleanPointer allocatedOutsideTlab) {
330344
assert size.equal(LayoutEncoding.getPureInstanceAllocationSize(hub.getLayoutEncoding()));
331-
Pointer memory = allocateRawMemory(size);
345+
Pointer memory = allocateRawMemory(size, allocatedOutsideTlab);
332346
return FormatObjectNode.formatObject(memory, DynamicHub.toClass(hub), false, FillContent.WITH_ZEROES, true);
333347
}
334348

@@ -342,19 +356,19 @@ private static Object allocateSmallArrayLikeObjectInCurrentTlab(DynamicHub hub,
342356
}
343357

344358
@Uninterruptible(reason = "Holds uninitialized memory.")
345-
private static Object allocateArraySlow(DynamicHub hub, int length, UnsignedWord size, byte[] podReferenceMap) {
346-
Pointer memory = allocateRawMemory(size);
359+
private static Object allocateArraySlow(DynamicHub hub, int length, UnsignedWord size, byte[] podReferenceMap, BooleanPointer allocatedOutsideTlab) {
360+
Pointer memory = allocateRawMemory(size, allocatedOutsideTlab);
347361
return formatArrayLikeObject(memory, hub, length, false, FillContent.WITH_ZEROES, podReferenceMap);
348362
}
349363

350364
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/memAllocator.cpp#L333-L341")
351365
@Uninterruptible(reason = "Holds uninitialized memory.")
352-
private static Pointer allocateRawMemory(UnsignedWord size) {
366+
private static Pointer allocateRawMemory(UnsignedWord size, BooleanPointer allocatedOutsideTlab) {
353367
Pointer memory = allocateRawMemoryInTlabSlow(size);
354368
if (memory.isNonNull()) {
355369
return memory;
356370
}
357-
return allocateRawMemoryOutsideTlab(size);
371+
return allocateRawMemoryOutsideTlab(size, allocatedOutsideTlab);
358372
}
359373

360374
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+8/src/hotspot/share/gc/shared/memAllocator.cpp#L256-L318")
@@ -404,7 +418,8 @@ private static Pointer allocateRawMemoryInTlabSlow(UnsignedWord size) {
404418

405419
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/memAllocator.cpp#L240-L251")
406420
@Uninterruptible(reason = "Holds uninitialized memory.")
407-
private static Pointer allocateRawMemoryOutsideTlab(UnsignedWord size) {
421+
private static Pointer allocateRawMemoryOutsideTlab(UnsignedWord size, BooleanPointer allocatedOutsideTlab) {
422+
allocatedOutsideTlab.write(true);
408423
Pointer memory = YoungGeneration.getHeapAllocation().allocateOutsideTlab(size);
409424
allocatedAlignedBytes.set(allocatedAlignedBytes.get().add(size));
410425
return memory;

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ public static void initialize(HostedByteBufferPointer chunk, UnsignedWord object
100100
RememberedSet.get().setObjectStartOffsetOfUnalignedChunk(chunk, objectStartOffset);
101101
}
102102

103+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
103104
public static void initialize(UnalignedHeader chunk, UnsignedWord chunkSize, UnsignedWord objectSize) {
104105
assert chunk.isNonNull();
105106
UnsignedWord objectStartOffset = calculateObjectStartOffset(objectSize);
@@ -116,6 +117,7 @@ public static Pointer getObjectEnd(UnalignedHeader that) {
116117
return HeapChunk.getEndPointer(that);
117118
}
118119

120+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
119121
static UnsignedWord getChunkSizeForObject(UnsignedWord objectSize) {
120122
UnsignedWord objectStart = RememberedSet.get().getHeaderSizeOfUnalignedChunk(objectSize);
121123
UnsignedWord alignment = Word.unsigned(ConfigurationValues.getObjectLayout().getAlignment());
@@ -155,10 +157,12 @@ public static void initializeObjectStartOffset(UnalignedHeader that, UnsignedWor
155157
setObjectStartOffset(that, objectStartOffset);
156158
}
157159

160+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
158161
public static UnsignedWord calculateObjectStartOffset(UnsignedWord objectSize) {
159162
return RememberedSet.get().getHeaderSizeOfUnalignedChunk(objectSize);
160163
}
161164

165+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
162166
public static void setObjectStartOffset(UnalignedHeader that, UnsignedWord objectStartOffset) {
163167
RememberedSet.get().setObjectStartOffsetOfUnalignedChunk(that, objectStartOffset);
164168
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSupport.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,14 @@
2424
*/
2525
package com.oracle.svm.core.genscavenge.graal;
2626

27+
import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
2728
import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.NO_SIDE_EFFECT;
2829

2930
import org.graalvm.word.UnsignedWord;
3031

32+
import com.oracle.svm.core.SubstrateGCOptions;
3133
import com.oracle.svm.core.Uninterruptible;
3234
import com.oracle.svm.core.genscavenge.HeapImpl;
33-
import com.oracle.svm.core.SubstrateGCOptions;
3435
import com.oracle.svm.core.genscavenge.HeapParameters;
3536
import com.oracle.svm.core.genscavenge.ThreadLocalAllocation;
3637
import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
@@ -123,6 +124,7 @@ public int tlabEndOffset() {
123124
return ThreadLocalAllocation.Descriptor.offsetOfAllocationEnd();
124125
}
125126

127+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
126128
public static boolean arrayAllocatedInAlignedChunk(UnsignedWord objectSize) {
127129
return objectSize.belowThan(HeapParameters.getLargeArrayThreshold());
128130
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ public UnsignedWord getHeaderSizeOfAlignedChunk() {
8282
}
8383

8484
@Override
85+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
8586
public UnsignedWord getHeaderSizeOfUnalignedChunk(UnsignedWord objectSize) {
8687
return UnalignedChunkRememberedSet.getHeaderSize(objectSize);
8788
}
@@ -93,6 +94,7 @@ public void setObjectStartOffsetOfUnalignedChunk(HostedByteBufferPointer chunk,
9394
}
9495

9596
@Override
97+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
9698
public void setObjectStartOffsetOfUnalignedChunk(UnalignedHeader chunk, UnsignedWord objectStartOffset) {
9799
UnalignedChunkRememberedSet.setObjectStartOffset(chunk, objectStartOffset);
98100
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ public UnsignedWord getHeaderSizeOfAlignedChunk() {
7070
}
7171

7272
@Override
73+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
7374
public UnsignedWord getHeaderSizeOfUnalignedChunk(UnsignedWord objectSize) {
7475
return getHeaderSizeOfUnalignedChunk();
7576
}
@@ -88,6 +89,7 @@ public void setObjectStartOffsetOfUnalignedChunk(HostedByteBufferPointer chunk,
8889
}
8990

9091
@Override
92+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
9193
public void setObjectStartOffsetOfUnalignedChunk(UnalignedHeader chunk, UnsignedWord objectStartOffset) {
9294
assert objectStartOffset.equal(getHeaderSizeOfUnalignedChunk());
9395
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,13 +61,15 @@ static RememberedSet get() {
6161
UnsignedWord getHeaderSizeOfAlignedChunk();
6262

6363
/** Returns the header size of an unaligned chunk for a given object size. */
64+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
6465
UnsignedWord getHeaderSizeOfUnalignedChunk(UnsignedWord objectSize);
6566

6667
/** Sets the object start offset in the unaligned chunk. */
6768
@Platforms(Platform.HOSTED_ONLY.class)
6869
void setObjectStartOffsetOfUnalignedChunk(HostedByteBufferPointer chunk, UnsignedWord objectStartOffset);
6970

7071
/** Sets the object start offset in the unaligned chunk. */
72+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
7173
void setObjectStartOffsetOfUnalignedChunk(UnalignedHeader chunk, UnsignedWord objectStartOffset);
7274

7375
/**

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ final class UnalignedChunkRememberedSet {
6565
private UnalignedChunkRememberedSet() {
6666
}
6767

68+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
6869
public static UnsignedWord getHeaderSize(UnsignedWord objectSize) {
6970
UnsignedWord headerSize = getCardTableLimitOffset(objectSize);
7071
headerSize = headerSize.add(sizeOfObjectStartOffsetField());
@@ -78,6 +79,7 @@ public static void setObjectStartOffset(HostedByteBufferPointer chunk, UnsignedW
7879
chunk.writeWord(objectStartOffset.subtract(sizeOfObjectStartOffsetField()), objectStartOffset);
7980
}
8081

82+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
8183
public static void setObjectStartOffset(UnalignedHeader chunk, UnsignedWord objectStartOffset) {
8284
HeapChunk.asPointer(chunk).writeWord(objectStartOffset.subtract(sizeOfObjectStartOffsetField()), objectStartOffset);
8385
assert getObjectStartOffset(chunk).equal(objectStartOffset);
@@ -385,6 +387,7 @@ private static UnsignedWord getCardTableSize(Pointer obj) {
385387
return getOffsetForObject(obj).subtract(sizeOfObjectStartOffsetField()).subtract(getCardTableStartOffset());
386388
}
387389

390+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
388391
private static UnsignedWord getCardTableLimitOffset(UnsignedWord objectSize) {
389392
UnsignedWord tableStart = getCardTableStartOffset();
390393
UnsignedWord tableSize = getCardTableSize(objectSize);

‎substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrEvent.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ public final class JfrEvent {
6565
public static final JfrEvent JavaMonitorWait = create("jdk.JavaMonitorWait", 5, JfrEventFlags.HasDuration);
6666
public static final JfrEvent JavaMonitorInflate = create("jdk.JavaMonitorInflate", 5, JfrEventFlags.HasDuration);
6767
public static final JfrEvent ObjectAllocationInNewTLAB = create("jdk.ObjectAllocationInNewTLAB", 5);
68+
public static final JfrEvent ObjectAllocationOutsideTLAB = create("jdk.ObjectAllocationOutsideTLAB", 5);
6869
public static final JfrEvent GCHeapSummary = create("jdk.GCHeapSummary");
6970
public static final JfrEvent ThreadAllocationStatistics = create("jdk.ThreadAllocationStatistics");
7071
public static final JfrEvent SystemGC = create("jdk.SystemGC", 5, JfrEventFlags.HasDuration);

0 commit comments

Comments
 (0)