Skip to content

Commit 1ada11c

Browse files
author
duke
committed
Backport 37ec796255ae857588a5c7e0d572407dd81cbec9
1 parent 186c4f0 commit 1ada11c

File tree

5 files changed

+42
-36
lines changed

5 files changed

+42
-36
lines changed

‎src/hotspot/share/gc/g1/g1Allocator.cpp‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -213,24 +213,24 @@ size_t G1Allocator::used_in_alloc_regions() {
213213

214214

215215
HeapWord* G1Allocator::par_allocate_during_gc(G1HeapRegionAttr dest,
216-
size_t word_size,
217-
uint node_index) {
216+
uint node_index,
217+
size_t word_size) {
218218
size_t temp = 0;
219-
HeapWord* result = par_allocate_during_gc(dest, word_size, word_size, &temp, node_index);
219+
HeapWord* result = par_allocate_during_gc(dest, node_index, word_size, word_size, &temp);
220220
assert(result == nullptr || temp == word_size,
221221
"Requested " SIZE_FORMAT " words, but got " SIZE_FORMAT " at " PTR_FORMAT,
222222
word_size, temp, p2i(result));
223223
return result;
224224
}
225225

226226
HeapWord* G1Allocator::par_allocate_during_gc(G1HeapRegionAttr dest,
227+
uint node_index,
227228
size_t min_word_size,
228229
size_t desired_word_size,
229-
size_t* actual_word_size,
230-
uint node_index) {
230+
size_t* actual_word_size) {
231231
switch (dest.type()) {
232232
case G1HeapRegionAttr::Young:
233-
return survivor_attempt_allocation(min_word_size, desired_word_size, actual_word_size, node_index);
233+
return survivor_attempt_allocation(node_index, min_word_size, desired_word_size, actual_word_size);
234234
case G1HeapRegionAttr::Old:
235235
return old_attempt_allocation(min_word_size, desired_word_size, actual_word_size);
236236
default:
@@ -239,10 +239,10 @@ HeapWord* G1Allocator::par_allocate_during_gc(G1HeapRegionAttr dest,
239239
}
240240
}
241241

242-
HeapWord* G1Allocator::survivor_attempt_allocation(size_t min_word_size,
242+
HeapWord* G1Allocator::survivor_attempt_allocation(uint node_index,
243+
size_t min_word_size,
243244
size_t desired_word_size,
244-
size_t* actual_word_size,
245-
uint node_index) {
245+
size_t* actual_word_size) {
246246
assert(!_g1h->is_humongous(desired_word_size),
247247
"we should not be seeing humongous-size allocations in this path");
248248

@@ -398,10 +398,10 @@ HeapWord* G1PLABAllocator::allocate_direct_or_new_plab(G1HeapRegionAttr dest,
398398

399399
size_t actual_plab_size = 0;
400400
HeapWord* buf = _allocator->par_allocate_during_gc(dest,
401+
node_index,
401402
required_in_plab,
402403
plab_word_size,
403-
&actual_plab_size,
404-
node_index);
404+
&actual_plab_size);
405405

406406
assert(buf == nullptr || ((actual_plab_size >= required_in_plab) && (actual_plab_size <= plab_word_size)),
407407
"Requested at minimum %zu, desired %zu words, but got %zu at " PTR_FORMAT,
@@ -420,7 +420,7 @@ HeapWord* G1PLABAllocator::allocate_direct_or_new_plab(G1HeapRegionAttr dest,
420420
*plab_refill_failed = true;
421421
}
422422
// Try direct allocation.
423-
HeapWord* result = _allocator->par_allocate_during_gc(dest, word_sz, node_index);
423+
HeapWord* result = _allocator->par_allocate_during_gc(dest, node_index, word_sz);
424424
if (result != nullptr) {
425425
plab_data->_direct_allocated += word_sz;
426426
plab_data->_num_direct_allocations++;

‎src/hotspot/share/gc/g1/g1Allocator.hpp‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -78,19 +78,16 @@ class G1Allocator : public CHeapObj<mtGC> {
7878
inline OldGCAllocRegion* old_gc_alloc_region();
7979

8080
// Allocation attempt during GC for a survivor object / PLAB.
81-
HeapWord* survivor_attempt_allocation(size_t min_word_size,
81+
HeapWord* survivor_attempt_allocation(uint node_index,
82+
size_t min_word_size,
8283
size_t desired_word_size,
83-
size_t* actual_word_size,
84-
uint node_index);
84+
size_t* actual_word_size);
8585

8686
// Allocation attempt during GC for an old object / PLAB.
8787
HeapWord* old_attempt_allocation(size_t min_word_size,
8888
size_t desired_word_size,
8989
size_t* actual_word_size);
9090

91-
// Node index of current thread.
92-
inline uint current_node_index() const;
93-
9491
public:
9592
G1Allocator(G1CollectedHeap* heap);
9693
~G1Allocator();
@@ -110,16 +107,20 @@ class G1Allocator : public CHeapObj<mtGC> {
110107
void abandon_gc_alloc_regions();
111108
bool is_retained_old_region(G1HeapRegion* hr);
112109

110+
// Node index of current thread.
111+
inline uint current_node_index() const;
112+
113113
// Allocate blocks of memory during mutator time.
114114

115115
// Attempt allocation in the current alloc region.
116-
inline HeapWord* attempt_allocation(size_t min_word_size,
116+
inline HeapWord* attempt_allocation(uint node_index,
117+
size_t min_word_size,
117118
size_t desired_word_size,
118119
size_t* actual_word_size);
119120

120121
// This is to be called when holding an appropriate lock. It first tries in the
121122
// current allocation region, and then attempts an allocation using a new region.
122-
inline HeapWord* attempt_allocation_locked(size_t word_size);
123+
inline HeapWord* attempt_allocation_locked(uint node_index, size_t word_size);
123124

124125
size_t unsafe_max_tlab_alloc();
125126
size_t used_in_alloc_regions();
@@ -129,14 +130,15 @@ class G1Allocator : public CHeapObj<mtGC> {
129130
// heap, and then allocate a block of the given size. The block
130131
// may not be a humongous - it must fit into a single heap region.
131132
HeapWord* par_allocate_during_gc(G1HeapRegionAttr dest,
132-
size_t word_size,
133-
uint node_index);
133+
uint node_index,
134+
size_t word_size
135+
);
134136

135137
HeapWord* par_allocate_during_gc(G1HeapRegionAttr dest,
138+
uint node_index,
136139
size_t min_word_size,
137140
size_t desired_word_size,
138-
size_t* actual_word_size,
139-
uint node_index);
141+
size_t* actual_word_size);
140142
};
141143

142144
// Manages the PLABs used during garbage collection. Interface for allocation from PLABs.

‎src/hotspot/share/gc/g1/g1Allocator.inline.hpp‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,10 @@ inline OldGCAllocRegion* G1Allocator::old_gc_alloc_region() {
4949
return &_old_gc_alloc_region;
5050
}
5151

52-
inline HeapWord* G1Allocator::attempt_allocation(size_t min_word_size,
52+
inline HeapWord* G1Allocator::attempt_allocation(uint node_index,
53+
size_t min_word_size,
5354
size_t desired_word_size,
5455
size_t* actual_word_size) {
55-
uint node_index = current_node_index();
56-
5756
HeapWord* result = mutator_alloc_region(node_index)->attempt_retained_allocation(min_word_size, desired_word_size, actual_word_size);
5857
if (result != nullptr) {
5958
return result;
@@ -62,8 +61,7 @@ inline HeapWord* G1Allocator::attempt_allocation(size_t min_word_size,
6261
return mutator_alloc_region(node_index)->attempt_allocation(min_word_size, desired_word_size, actual_word_size);
6362
}
6463

65-
inline HeapWord* G1Allocator::attempt_allocation_locked(size_t word_size) {
66-
uint node_index = current_node_index();
64+
inline HeapWord* G1Allocator::attempt_allocation_locked(uint node_index, size_t word_size) {
6765
HeapWord* result = mutator_alloc_region(node_index)->attempt_allocation_locked(word_size);
6866

6967
assert(result != nullptr || mutator_alloc_region(node_index)->get() == nullptr,

‎src/hotspot/share/gc/g1/g1CollectedHeap.cpp‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,7 @@ G1CollectedHeap::mem_allocate(size_t word_size,
401401
return attempt_allocation(word_size, word_size, &dummy);
402402
}
403403

404-
HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size) {
404+
HeapWord* G1CollectedHeap::attempt_allocation_slow(uint node_index, size_t word_size) {
405405
ResourceMark rm; // For retrieving the thread names in log messages.
406406

407407
// Make sure you read the note in attempt_allocation_humongous().
@@ -425,7 +425,7 @@ HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size) {
425425

426426
// Now that we have the lock, we first retry the allocation in case another
427427
// thread changed the region while we were waiting to acquire the lock.
428-
result = _allocator->attempt_allocation_locked(word_size);
428+
result = _allocator->attempt_allocation_locked(node_index, word_size);
429429
if (result != nullptr) {
430430
return result;
431431
}
@@ -452,7 +452,7 @@ HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size) {
452452
// here and the follow-on attempt will be at the start of the next loop
453453
// iteration (after taking the Heap_lock).
454454
size_t dummy = 0;
455-
result = _allocator->attempt_allocation(word_size, word_size, &dummy);
455+
result = _allocator->attempt_allocation(node_index, word_size, word_size, &dummy);
456456
if (result != nullptr) {
457457
return result;
458458
}
@@ -586,11 +586,14 @@ inline HeapWord* G1CollectedHeap::attempt_allocation(size_t min_word_size,
586586
assert(!is_humongous(desired_word_size), "attempt_allocation() should not "
587587
"be called for humongous allocation requests");
588588

589-
HeapWord* result = _allocator->attempt_allocation(min_word_size, desired_word_size, actual_word_size);
589+
// Fix NUMA node association for the duration of this allocation
590+
const uint node_index = _allocator->current_node_index();
591+
592+
HeapWord* result = _allocator->attempt_allocation(node_index, min_word_size, desired_word_size, actual_word_size);
590593

591594
if (result == nullptr) {
592595
*actual_word_size = desired_word_size;
593-
result = attempt_allocation_slow(desired_word_size);
596+
result = attempt_allocation_slow(node_index, desired_word_size);
594597
}
595598

596599
assert_heap_not_locked();
@@ -697,8 +700,11 @@ HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size,
697700
assert(!_allocator->has_mutator_alloc_region() || !expect_null_mutator_alloc_region,
698701
"the current alloc region was unexpectedly found to be non-null");
699702

703+
// Fix NUMA node association for the duration of this allocation
704+
const uint node_index = _allocator->current_node_index();
705+
700706
if (!is_humongous(word_size)) {
701-
return _allocator->attempt_allocation_locked(word_size);
707+
return _allocator->attempt_allocation_locked(node_index, word_size);
702708
} else {
703709
HeapWord* result = humongous_obj_allocate(word_size);
704710
if (result != nullptr && policy()->need_to_start_conc_mark("STW humongous allocation")) {

‎src/hotspot/share/gc/g1/g1CollectedHeap.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -449,7 +449,7 @@ class G1CollectedHeap : public CollectedHeap {
449449
// Second-level mutator allocation attempt: take the Heap_lock and
450450
// retry the allocation attempt, potentially scheduling a GC
451451
// pause. This should only be used for non-humongous allocations.
452-
HeapWord* attempt_allocation_slow(size_t word_size);
452+
HeapWord* attempt_allocation_slow(uint node_index, size_t word_size);
453453

454454
// Takes the Heap_lock and attempts a humongous allocation. It can
455455
// potentially schedule a GC pause.

0 commit comments

Comments
 (0)