Skip to content

Commit 0f74838

Browse files
Thomas Schrottchristianhaeubl
Thomas Schrott
authored and
christianhaeubl
committed
[GR-32941] Allocate TLABs from chunks instead of as chunks
PullRequest: graal/19278
2 parents ea31dbc + ef18876 commit 0f74838

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1898
-373
lines changed

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveWeightedAverage.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,11 @@
2424
*/
2525
package com.oracle.svm.core.genscavenge;
2626

27+
import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
28+
2729
import org.graalvm.word.UnsignedWord;
2830

31+
import com.oracle.svm.core.Uninterruptible;
2932
import com.oracle.svm.core.util.UnsignedUtils;
3033

3134
/**
@@ -101,7 +104,8 @@ protected double computeAdaptiveAverage(double sample, double avg) {
101104
return expAvg(avg, sample, adaptiveWeight);
102105
}
103106

104-
private static double expAvg(double avg, double sample, double adaptiveWeight) {
107+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
108+
static double expAvg(double avg, double sample, double adaptiveWeight) {
105109
assert adaptiveWeight > 0 && adaptiveWeight <= 100 : "weight must be a percentage";
106110
return (100.0 - adaptiveWeight) * avg / 100.0 + adaptiveWeight * sample / 100.0;
107111
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveWeightedAverageStruct.java‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/*
2+
* Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* This code is free software; you can redistribute it and/or modify it
6+
* under the terms of the GNU General Public License version 2 only, as
7+
* published by the Free Software Foundation. Oracle designates this
8+
* particular file as subject to the "Classpath" exception as provided
9+
* by Oracle in the LICENSE file that accompanied this code.
10+
*
11+
* This code is distributed in the hope that it will be useful, but WITHOUT
12+
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13+
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14+
* version 2 for more details (a copy is included in the LICENSE file that
15+
* accompanied this code).
16+
*
17+
* You should have received a copy of the GNU General Public License version
18+
* 2 along with this work; if not, write to the Free Software Foundation,
19+
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20+
*
21+
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22+
* or visit www.oracle.com if you need additional information or have any
23+
* questions.
24+
*/
25+
package com.oracle.svm.core.genscavenge;
26+
27+
import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
28+
import static com.oracle.svm.core.genscavenge.AdaptiveWeightedAverage.OLD_THRESHOLD;
29+
30+
import org.graalvm.nativeimage.c.struct.RawField;
31+
import org.graalvm.nativeimage.c.struct.RawStructure;
32+
import org.graalvm.word.PointerBase;
33+
import org.graalvm.word.UnsignedWord;
34+
35+
import com.oracle.svm.core.Uninterruptible;
36+
import com.oracle.svm.core.jdk.UninterruptibleUtils;
37+
import com.oracle.svm.core.util.UnsignedUtils;
38+
39+
/**
40+
* This class provides a raw structure implementation of {@link AdaptiveWeightedAverage}. For
41+
* further information see {@link AdaptiveWeightedAverage}.
42+
*/
43+
class AdaptiveWeightedAverageStruct {
44+
45+
@RawStructure
46+
interface Data extends PointerBase {
47+
48+
@RawField
49+
void setWeight(double weight);
50+
51+
@RawField
52+
double getWeight();
53+
54+
@RawField
55+
void setAverage(double average);
56+
57+
@RawField
58+
double getAverage();
59+
60+
@RawField
61+
void setSampleCount(long sampleCount);
62+
63+
@RawField
64+
long getSampleCount();
65+
66+
@RawField
67+
void setIsOld(boolean isOld);
68+
69+
@RawField
70+
boolean getIsOld();
71+
}
72+
73+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
74+
static void initialize(Data data, double weight) {
75+
initialize(data, weight, 0);
76+
}
77+
78+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
79+
static void initialize(Data data, double weight, double avg) {
80+
assert weight > 0 && weight <= 100;
81+
data.setWeight(weight);
82+
data.setAverage(avg);
83+
}
84+
85+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
86+
public static double getAverage(Data data) {
87+
return data.getAverage();
88+
}
89+
90+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
91+
public static void sample(Data data, double value) {
92+
data.setSampleCount(data.getSampleCount() + 1);
93+
if (!data.getIsOld() && data.getSampleCount() > OLD_THRESHOLD) {
94+
data.setIsOld(true);
95+
}
96+
data.setAverage(computeAdaptiveAverage(data, value, data.getAverage()));
97+
}
98+
99+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
100+
public static void sample(Data data, UnsignedWord value) {
101+
sample(data, UnsignedUtils.toDouble(value));
102+
}
103+
104+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
105+
protected static double computeAdaptiveAverage(Data data, double sample, double avg) {
106+
/*
107+
* We smoothen the samples by not using weight directly until we've had enough data to make
108+
* it meaningful. We'd like the first weight used to be 1, the second to be 1/2, etc until
109+
* we have OLD_THRESHOLD/weight samples.
110+
*/
111+
double countWeight = 0;
112+
if (!data.getIsOld()) { // avoid division by zero if the counter wraps
113+
countWeight = OLD_THRESHOLD / (double) data.getSampleCount();
114+
}
115+
double adaptiveWeight = UninterruptibleUtils.Math.max(data.getWeight(), countWeight);
116+
return AdaptiveWeightedAverage.expAvg(avg, sample, adaptiveWeight);
117+
}
118+
119+
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AddressRangeCommittedMemoryProvider.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,7 @@ protected OutOfMemoryError reportAlignedChunkAllocationFailed(int error) {
351351
}
352352

353353
@Override
354+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
354355
public Pointer allocateUnalignedChunk(UnsignedWord nbytes) {
355356
WordPointer allocOut = UnsafeStackValue.get(WordPointer.class);
356357
int error = allocateInHeapAddressSpace(nbytes, getAlignmentForUnalignedChunks(), allocOut);

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -221,10 +221,7 @@ public static final class OnlyCompletely extends BasicPolicy {
221221

222222
@Override
223223
public boolean shouldCollectCompletely(boolean followingIncrementalCollection) {
224-
if (!followingIncrementalCollection && shouldCollectYoungGenSeparately(false)) {
225-
return false;
226-
}
227-
return true;
224+
return followingIncrementalCollection || !shouldCollectYoungGenSeparately(false);
228225
}
229226

230227
@Override

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/FillerObjectUtil.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* This code is free software; you can redistribute it and/or modify it
6+
* under the terms of the GNU General Public License version 2 only, as
7+
* published by the Free Software Foundation. Oracle designates this
8+
* particular file as subject to the "Classpath" exception as provided
9+
* by Oracle in the LICENSE file that accompanied this code.
10+
*
11+
* This code is distributed in the hope that it will be useful, but WITHOUT
12+
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13+
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14+
* version 2 for more details (a copy is included in the LICENSE file that
15+
* accompanied this code).
16+
*
17+
* You should have received a copy of the GNU General Public License version
18+
* 2 along with this work; if not, write to the Free Software Foundation,
19+
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20+
*
21+
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22+
* or visit www.oracle.com if you need additional information or have any
23+
* questions.
24+
*/
25+
package com.oracle.svm.core.genscavenge;
26+
27+
import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
28+
import static jdk.graal.compiler.replacements.AllocationSnippets.FillContent.WITH_GARBAGE_IF_ASSERTIONS_ENABLED;
29+
30+
import org.graalvm.word.Pointer;
31+
import org.graalvm.word.UnsignedWord;
32+
33+
import com.oracle.svm.core.Uninterruptible;
34+
import com.oracle.svm.core.config.ConfigurationValues;
35+
import com.oracle.svm.core.genscavenge.graal.nodes.FormatArrayNode;
36+
import com.oracle.svm.core.genscavenge.graal.nodes.FormatObjectNode;
37+
import com.oracle.svm.core.heap.FillerArray;
38+
import com.oracle.svm.core.heap.FillerObject;
39+
import com.oracle.svm.core.hub.LayoutEncoding;
40+
import com.oracle.svm.core.util.UnsignedUtils;
41+
42+
import jdk.graal.compiler.api.replacements.Fold;
43+
import jdk.graal.compiler.core.common.NumUtil;
44+
import jdk.graal.compiler.word.Word;
45+
import jdk.vm.ci.meta.JavaKind;
46+
47+
public class FillerObjectUtil {
48+
private static final Class<?> ARRAY_CLASS = FillerArray.class;
49+
private static final JavaKind ARRAY_ELEMENT_KIND = JavaKind.Int;
50+
private static final int ARRAY_ELEMENT_SIZE = ARRAY_ELEMENT_KIND.getByteCount();
51+
52+
@Fold
53+
public static UnsignedWord objectMinSize() {
54+
return Word.unsigned(ConfigurationValues.getObjectLayout().getMinImageHeapObjectSize());
55+
}
56+
57+
@Fold
58+
static int arrayMinSize() {
59+
return NumUtil.safeToInt(ConfigurationValues.getObjectLayout().getArraySize(ARRAY_ELEMENT_KIND, 0, false));
60+
}
61+
62+
@Fold
63+
static int arrayBaseOffset() {
64+
return ConfigurationValues.getObjectLayout().getArrayBaseOffset(ARRAY_ELEMENT_KIND);
65+
}
66+
67+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
68+
public static void writeFillerObjectAt(Pointer p, UnsignedWord size) {
69+
assert size.aboveThan(0);
70+
if (size.aboveOrEqual(arrayMinSize())) {
71+
int length = UnsignedUtils.safeToInt(size.subtract(arrayBaseOffset()).unsignedDivide(ARRAY_ELEMENT_SIZE));
72+
FormatArrayNode.formatArray(p, ARRAY_CLASS, length, true, false, WITH_GARBAGE_IF_ASSERTIONS_ENABLED, false);
73+
} else {
74+
FormatObjectNode.formatObject(p, FillerObject.class, true, WITH_GARBAGE_IF_ASSERTIONS_ENABLED, false);
75+
}
76+
assert LayoutEncoding.getSizeFromObjectInGC(p.toObject()).equal(size);
77+
}
78+
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,8 @@ assert getCollectionEpoch().equal(data.getRequestingEpoch()) ||
245245

246246
Timer collectionTimer = timers.collection.start();
247247
try {
248-
ThreadLocalAllocation.disableAndFlushForAllThreads();
248+
HeapImpl.getHeapImpl().makeParseable();
249+
249250
GenScavengeMemoryPoolMXBeans.singleton().notifyBeforeCollection();
250251
HeapImpl.getAccounting().notifyBeforeCollection();
251252

@@ -259,6 +260,7 @@ assert getCollectionEpoch().equal(data.getRequestingEpoch()) ||
259260
collectionTimer.stop();
260261
}
261262

263+
resizeAllTlabs();
262264
accounting.updateCollectionCountAndTime(completeCollection, collectionTimer.totalNanos());
263265
HeapImpl.getAccounting().notifyAfterCollection();
264266
GenScavengeMemoryPoolMXBeans.singleton().notifyAfterCollection();
@@ -401,6 +403,14 @@ public static UnsignedWord getChunkBytes() {
401403
return youngBytes.add(oldBytes);
402404
}
403405

406+
private static void resizeAllTlabs() {
407+
if (SubstrateGCOptions.TlabOptions.ResizeTLAB.getValue()) {
408+
for (IsolateThread thread = VMThreads.firstThread(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) {
409+
TlabSupport.resize(thread);
410+
}
411+
}
412+
}
413+
404414
private void printGCBefore(GCCause cause) {
405415
if (!SubstrateGCOptions.VerboseGC.getValue()) {
406416
return;
@@ -1316,7 +1326,7 @@ protected PrintGCSummaryOperation() {
13161326

13171327
@Override
13181328
protected void operate() {
1319-
ThreadLocalAllocation.disableAndFlushForAllThreads();
1329+
HeapImpl.getHeapImpl().makeParseable();
13201330

13211331
Log log = Log.log();
13221332
log.string("GC summary").indent(true);

0 commit comments

Comments
 (0)