Skip to content

Commit 0aebb17

Browse files
author
duke
committed
Backport 37ec796255ae857588a5c7e0d572407dd81cbec9
1 parent 4d3a3c0 commit 0aebb17

File tree

5 files changed

+42
-36
lines changed

5 files changed

+42
-36
lines changed

‎src/hotspot/share/gc/g1/g1Allocator.cpp‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -212,24 +212,24 @@ size_t G1Allocator::used_in_alloc_regions() {
212212

213213

214214
HeapWord* G1Allocator::par_allocate_during_gc(G1HeapRegionAttr dest,
215-
size_t word_size,
216-
uint node_index) {
215+
uint node_index,
216+
size_t word_size) {
217217
size_t temp = 0;
218-
HeapWord* result = par_allocate_during_gc(dest, word_size, word_size, &temp, node_index);
218+
HeapWord* result = par_allocate_during_gc(dest, node_index, word_size, word_size, &temp);
219219
assert(result == nullptr || temp == word_size,
220220
"Requested " SIZE_FORMAT " words, but got " SIZE_FORMAT " at " PTR_FORMAT,
221221
word_size, temp, p2i(result));
222222
return result;
223223
}
224224

225225
HeapWord* G1Allocator::par_allocate_during_gc(G1HeapRegionAttr dest,
226+
uint node_index,
226227
size_t min_word_size,
227228
size_t desired_word_size,
228-
size_t* actual_word_size,
229-
uint node_index) {
229+
size_t* actual_word_size) {
230230
switch (dest.type()) {
231231
case G1HeapRegionAttr::Young:
232-
return survivor_attempt_allocation(min_word_size, desired_word_size, actual_word_size, node_index);
232+
return survivor_attempt_allocation(node_index, min_word_size, desired_word_size, actual_word_size);
233233
case G1HeapRegionAttr::Old:
234234
return old_attempt_allocation(min_word_size, desired_word_size, actual_word_size);
235235
default:
@@ -238,10 +238,10 @@ HeapWord* G1Allocator::par_allocate_during_gc(G1HeapRegionAttr dest,
238238
}
239239
}
240240

241-
HeapWord* G1Allocator::survivor_attempt_allocation(size_t min_word_size,
241+
HeapWord* G1Allocator::survivor_attempt_allocation(uint node_index,
242+
size_t min_word_size,
242243
size_t desired_word_size,
243-
size_t* actual_word_size,
244-
uint node_index) {
244+
size_t* actual_word_size) {
245245
assert(!_g1h->is_humongous(desired_word_size),
246246
"we should not be seeing humongous-size allocations in this path");
247247

@@ -396,10 +396,10 @@ HeapWord* G1PLABAllocator::allocate_direct_or_new_plab(G1HeapRegionAttr dest,
396396

397397
size_t actual_plab_size = 0;
398398
HeapWord* buf = _allocator->par_allocate_during_gc(dest,
399+
node_index,
399400
required_in_plab,
400401
plab_word_size,
401-
&actual_plab_size,
402-
node_index);
402+
&actual_plab_size);
403403

404404
assert(buf == nullptr || ((actual_plab_size >= required_in_plab) && (actual_plab_size <= plab_word_size)),
405405
"Requested at minimum %zu, desired %zu words, but got %zu at " PTR_FORMAT,
@@ -418,7 +418,7 @@ HeapWord* G1PLABAllocator::allocate_direct_or_new_plab(G1HeapRegionAttr dest,
418418
*plab_refill_failed = true;
419419
}
420420
// Try direct allocation.
421-
HeapWord* result = _allocator->par_allocate_during_gc(dest, word_sz, node_index);
421+
HeapWord* result = _allocator->par_allocate_during_gc(dest, node_index, word_sz);
422422
if (result != nullptr) {
423423
plab_data->_direct_allocated += word_sz;
424424
plab_data->_num_direct_allocations++;

‎src/hotspot/share/gc/g1/g1Allocator.hpp‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -78,19 +78,16 @@ class G1Allocator : public CHeapObj<mtGC> {
7878
inline OldGCAllocRegion* old_gc_alloc_region();
7979

8080
// Allocation attempt during GC for a survivor object / PLAB.
81-
HeapWord* survivor_attempt_allocation(size_t min_word_size,
81+
HeapWord* survivor_attempt_allocation(uint node_index,
82+
size_t min_word_size,
8283
size_t desired_word_size,
83-
size_t* actual_word_size,
84-
uint node_index);
84+
size_t* actual_word_size);
8585

8686
// Allocation attempt during GC for an old object / PLAB.
8787
HeapWord* old_attempt_allocation(size_t min_word_size,
8888
size_t desired_word_size,
8989
size_t* actual_word_size);
9090

91-
// Node index of current thread.
92-
inline uint current_node_index() const;
93-
9491
public:
9592
G1Allocator(G1CollectedHeap* heap);
9693
~G1Allocator();
@@ -110,16 +107,20 @@ class G1Allocator : public CHeapObj<mtGC> {
110107
void abandon_gc_alloc_regions();
111108
bool is_retained_old_region(HeapRegion* hr);
112109

110+
// Node index of current thread.
111+
inline uint current_node_index() const;
112+
113113
// Allocate blocks of memory during mutator time.
114114

115115
// Attempt allocation in the current alloc region.
116-
inline HeapWord* attempt_allocation(size_t min_word_size,
116+
inline HeapWord* attempt_allocation(uint node_index,
117+
size_t min_word_size,
117118
size_t desired_word_size,
118119
size_t* actual_word_size);
119120

120121
// This is to be called when holding an appropriate lock. It first tries in the
121122
// current allocation region, and then attempts an allocation using a new region.
122-
inline HeapWord* attempt_allocation_locked(size_t word_size);
123+
inline HeapWord* attempt_allocation_locked(uint node_index, size_t word_size);
123124

124125
inline HeapWord* attempt_allocation_force(size_t word_size);
125126

@@ -131,14 +132,15 @@ class G1Allocator : public CHeapObj<mtGC> {
131132
// heap, and then allocate a block of the given size. The block
132133
// may not be a humongous - it must fit into a single heap region.
133134
HeapWord* par_allocate_during_gc(G1HeapRegionAttr dest,
134-
size_t word_size,
135-
uint node_index);
135+
uint node_index,
136+
size_t word_size
137+
);
136138

137139
HeapWord* par_allocate_during_gc(G1HeapRegionAttr dest,
140+
uint node_index,
138141
size_t min_word_size,
139142
size_t desired_word_size,
140-
size_t* actual_word_size,
141-
uint node_index);
143+
size_t* actual_word_size);
142144
};
143145

144146
// Manages the PLABs used during garbage collection. Interface for allocation from PLABs.

‎src/hotspot/share/gc/g1/g1Allocator.inline.hpp‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,10 @@ inline OldGCAllocRegion* G1Allocator::old_gc_alloc_region() {
4949
return &_old_gc_alloc_region;
5050
}
5151

52-
inline HeapWord* G1Allocator::attempt_allocation(size_t min_word_size,
52+
inline HeapWord* G1Allocator::attempt_allocation(uint node_index,
53+
size_t min_word_size,
5354
size_t desired_word_size,
5455
size_t* actual_word_size) {
55-
uint node_index = current_node_index();
56-
5756
HeapWord* result = mutator_alloc_region(node_index)->attempt_retained_allocation(min_word_size, desired_word_size, actual_word_size);
5857
if (result != nullptr) {
5958
return result;
@@ -62,8 +61,7 @@ inline HeapWord* G1Allocator::attempt_allocation(size_t min_word_size,
6261
return mutator_alloc_region(node_index)->attempt_allocation(min_word_size, desired_word_size, actual_word_size);
6362
}
6463

65-
inline HeapWord* G1Allocator::attempt_allocation_locked(size_t word_size) {
66-
uint node_index = current_node_index();
64+
inline HeapWord* G1Allocator::attempt_allocation_locked(uint node_index, size_t word_size) {
6765
HeapWord* result = mutator_alloc_region(node_index)->attempt_allocation_locked(word_size);
6866

6967
assert(result != nullptr || mutator_alloc_region(node_index)->get() == nullptr,

‎src/hotspot/share/gc/g1/g1CollectedHeap.cpp‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,7 @@ G1CollectedHeap::mem_allocate(size_t word_size,
401401
return attempt_allocation(word_size, word_size, &dummy);
402402
}
403403

404-
HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size) {
404+
HeapWord* G1CollectedHeap::attempt_allocation_slow(uint node_index, size_t word_size) {
405405
ResourceMark rm; // For retrieving the thread names in log messages.
406406

407407
// Make sure you read the note in attempt_allocation_humongous().
@@ -427,7 +427,7 @@ HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size) {
427427

428428
// Now that we have the lock, we first retry the allocation in case another
429429
// thread changed the region while we were waiting to acquire the lock.
430-
result = _allocator->attempt_allocation_locked(word_size);
430+
result = _allocator->attempt_allocation_locked(node_index, word_size);
431431
if (result != nullptr) {
432432
return result;
433433
}
@@ -495,7 +495,7 @@ HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size) {
495495
// follow-on attempt will be at the start of the next loop
496496
// iteration (after taking the Heap_lock).
497497
size_t dummy = 0;
498-
result = _allocator->attempt_allocation(word_size, word_size, &dummy);
498+
result = _allocator->attempt_allocation(node_index, word_size, word_size, &dummy);
499499
if (result != nullptr) {
500500
return result;
501501
}
@@ -636,11 +636,14 @@ inline HeapWord* G1CollectedHeap::attempt_allocation(size_t min_word_size,
636636
assert(!is_humongous(desired_word_size), "attempt_allocation() should not "
637637
"be called for humongous allocation requests");
638638

639-
HeapWord* result = _allocator->attempt_allocation(min_word_size, desired_word_size, actual_word_size);
639+
// Fix NUMA node association for the duration of this allocation
640+
const uint node_index = _allocator->current_node_index();
641+
642+
HeapWord* result = _allocator->attempt_allocation(node_index, min_word_size, desired_word_size, actual_word_size);
640643

641644
if (result == nullptr) {
642645
*actual_word_size = desired_word_size;
643-
result = attempt_allocation_slow(desired_word_size);
646+
result = attempt_allocation_slow(node_index, desired_word_size);
644647
}
645648

646649
assert_heap_not_locked();
@@ -778,8 +781,11 @@ HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size,
778781
assert(!_allocator->has_mutator_alloc_region() || !expect_null_mutator_alloc_region,
779782
"the current alloc region was unexpectedly found to be non-null");
780783

784+
// Fix NUMA node association for the duration of this allocation
785+
const uint node_index = _allocator->current_node_index();
786+
781787
if (!is_humongous(word_size)) {
782-
return _allocator->attempt_allocation_locked(word_size);
788+
return _allocator->attempt_allocation_locked(node_index, word_size);
783789
} else {
784790
HeapWord* result = humongous_obj_allocate(word_size);
785791
if (result != nullptr && policy()->need_to_start_conc_mark("STW humongous allocation")) {

‎src/hotspot/share/gc/g1/g1CollectedHeap.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -453,7 +453,7 @@ class G1CollectedHeap : public CollectedHeap {
453453
// Second-level mutator allocation attempt: take the Heap_lock and
454454
// retry the allocation attempt, potentially scheduling a GC
455455
// pause. This should only be used for non-humongous allocations.
456-
HeapWord* attempt_allocation_slow(size_t word_size);
456+
HeapWord* attempt_allocation_slow(uint node_index, size_t word_size);
457457

458458
// Takes the Heap_lock and attempts a humongous allocation. It can
459459
// potentially schedule a GC pause.

0 commit comments

Comments
 (0)