Skip to content

Commit d6eb8ac

Browse files
interwqinterwq
committed
Set reentrancy when invoking customized extent hooks.
Customized extent hooks may malloc / free thus trigger reentry. Support this behavior by adding reentrancy on hook functions.
1 parent d49ac4c commit d6eb8ac

File tree

5 files changed

+109
-35
lines changed

5 files changed

+109
-35
lines changed

‎include/jemalloc/internal/base_externs.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
base_t *b0get(void);
55
base_t *base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
6-
void base_delete(base_t *base);
6+
void base_delete(tsdn_t *tsdn, base_t *base);
77
extent_hooks_t *base_extent_hooks_get(base_t *base);
88
extent_hooks_t *base_extent_hooks_set(base_t *base,
99
extent_hooks_t *extent_hooks);

‎src/arena.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1257,7 +1257,7 @@ arena_destroy(tsd_t *tsd, arena_t *arena) {
12571257
* Destroy the base allocator, which manages all metadata ever mapped by
12581258
* this arena.
12591259
*/
1260-
base_delete(arena->base);
1260+
base_delete(tsd_tsdn(tsd), arena->base);
12611261
}
12621262

12631263
static extent_t *
@@ -2061,7 +2061,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
20612061
return arena;
20622062
label_error:
20632063
if (ind != 0) {
2064-
base_delete(base);
2064+
base_delete(tsdn, base);
20652065
}
20662066
return NULL;
20672067
}

‎src/base.c‎

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ static base_t *b0;
1515
/******************************************************************************/
1616

1717
static void *
18-
base_map(extent_hooks_t *extent_hooks, unsigned ind, size_t size) {
18+
base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size) {
1919
void *addr;
2020
bool zero = true;
2121
bool commit = true;
@@ -25,15 +25,18 @@ base_map(extent_hooks_t *extent_hooks, unsigned ind, size_t size) {
2525
if (extent_hooks == &extent_hooks_default) {
2626
addr = extent_alloc_mmap(NULL, size, PAGE, &zero, &commit);
2727
} else {
28+
assert(!tsdn_null(tsdn));
29+
pre_reentrancy(tsdn_tsd(tsdn));
2830
addr = extent_hooks->alloc(extent_hooks, NULL, size, PAGE,
2931
&zero, &commit, ind);
32+
post_reentrancy(tsdn_tsd(tsdn));
3033
}
3134

3235
return addr;
3336
}
3437

3538
static void
36-
base_unmap(extent_hooks_t *extent_hooks, unsigned ind, void *addr,
39+
base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr,
3740
size_t size) {
3841
/*
3942
* Cascade through dalloc, decommit, purge_forced, and purge_lazy,
@@ -61,27 +64,32 @@ base_unmap(extent_hooks_t *extent_hooks, unsigned ind, void *addr,
6164
/* Nothing worked. This should never happen. */
6265
not_reached();
6366
} else {
67+
assert(!tsdn_null(tsdn));
68+
pre_reentrancy(tsdn_tsd(tsdn));
6469
if (extent_hooks->dalloc != NULL &&
6570
!extent_hooks->dalloc(extent_hooks, addr, size, true,
6671
ind)) {
67-
return;
72+
goto label_done;
6873
}
6974
if (extent_hooks->decommit != NULL &&
7075
!extent_hooks->decommit(extent_hooks, addr, size, 0, size,
7176
ind)) {
72-
return;
77+
goto label_done;
7378
}
7479
if (extent_hooks->purge_forced != NULL &&
7580
!extent_hooks->purge_forced(extent_hooks, addr, size, 0,
7681
size, ind)) {
77-
return;
82+
goto label_done;
7883
}
7984
if (extent_hooks->purge_lazy != NULL &&
8085
!extent_hooks->purge_lazy(extent_hooks, addr, size, 0, size,
8186
ind)) {
82-
return;
87+
goto label_done;
8388
}
8489
/* Nothing worked. That's the application's problem. */
90+
label_done:
91+
post_reentrancy(tsdn_tsd(tsdn));
92+
return;
8593
}
8694
}
8795

@@ -157,7 +165,7 @@ base_extent_bump_alloc(tsdn_t *tsdn, base_t *base, extent_t *extent,
157165
* On success a pointer to the initialized base_block_t header is returned.
158166
*/
159167
static base_block_t *
160-
base_block_alloc(extent_hooks_t *extent_hooks, unsigned ind,
168+
base_block_alloc(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind,
161169
pszind_t *pind_last, size_t *extent_sn_next, size_t size,
162170
size_t alignment) {
163171
alignment = ALIGNMENT_CEILING(alignment, QUANTUM);
@@ -179,7 +187,7 @@ base_block_alloc(extent_hooks_t *extent_hooks, unsigned ind,
179187
size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next));
180188
size_t block_size = (min_block_size > next_block_size) ? min_block_size
181189
: next_block_size;
182-
base_block_t *block = (base_block_t *)base_map(extent_hooks, ind,
190+
base_block_t *block = (base_block_t *)base_map(tsdn, extent_hooks, ind,
183191
block_size);
184192
if (block == NULL) {
185193
return NULL;
@@ -207,8 +215,9 @@ base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
207215
* called.
208216
*/
209217
malloc_mutex_unlock(tsdn, &base->mtx);
210-
base_block_t *block = base_block_alloc(extent_hooks, base_ind_get(base),
211-
&base->pind_last, &base->extent_sn_next, size, alignment);
218+
base_block_t *block = base_block_alloc(tsdn, extent_hooks,
219+
base_ind_get(base), &base->pind_last, &base->extent_sn_next, size,
220+
alignment);
212221
malloc_mutex_lock(tsdn, &base->mtx);
213222
if (block == NULL) {
214223
return NULL;
@@ -234,8 +243,8 @@ base_t *
234243
base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
235244
pszind_t pind_last = 0;
236245
size_t extent_sn_next = 0;
237-
base_block_t *block = base_block_alloc(extent_hooks, ind, &pind_last,
238-
&extent_sn_next, sizeof(base_t), QUANTUM);
246+
base_block_t *block = base_block_alloc(tsdn, extent_hooks, ind,
247+
&pind_last, &extent_sn_next, sizeof(base_t), QUANTUM);
239248
if (block == NULL) {
240249
return NULL;
241250
}
@@ -249,7 +258,7 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
249258
atomic_store_p(&base->extent_hooks, extent_hooks, ATOMIC_RELAXED);
250259
if (malloc_mutex_init(&base->mtx, "base", WITNESS_RANK_BASE,
251260
malloc_mutex_rank_exclusive)) {
252-
base_unmap(extent_hooks, ind, block, block->size);
261+
base_unmap(tsdn, extent_hooks, ind, block, block->size);
253262
return NULL;
254263
}
255264
base->pind_last = pind_last;
@@ -272,13 +281,13 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
272281
}
273282

274283
void
275-
base_delete(base_t *base) {
284+
base_delete(tsdn_t *tsdn, base_t *base) {
276285
extent_hooks_t *extent_hooks = base_extent_hooks_get(base);
277286
base_block_t *next = base->blocks;
278287
do {
279288
base_block_t *block = next;
280289
next = block->next;
281-
base_unmap(extent_hooks, base_ind_get(base), block,
290+
base_unmap(tsdn, extent_hooks, base_ind_get(base), block,
282291
block->size);
283292
} while (next != NULL);
284293
}

‎src/extent.c‎

Lines changed: 76 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1073,9 +1073,12 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
10731073
&zeroed, &committed, (dss_prec_t)atomic_load_u(
10741074
&arena->dss_prec, ATOMIC_RELAXED));
10751075
} else {
1076+
assert(!tsdn_null(tsdn));
1077+
pre_reentrancy(tsdn_tsd(tsdn));
10761078
ptr = (*r_extent_hooks)->alloc(*r_extent_hooks, NULL,
10771079
alloc_size, PAGE, &zeroed, &committed,
10781080
arena_ind_get(arena));
1081+
post_reentrancy(tsdn_tsd(tsdn));
10791082
}
10801083

10811084
extent_init(extent, arena, ptr, alloc_size, false, NSIZES,
@@ -1247,8 +1250,11 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
12471250
addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize,
12481251
alignment, zero, commit);
12491252
} else {
1253+
assert(!tsdn_null(tsdn));
1254+
pre_reentrancy(tsdn_tsd(tsdn));
12501255
addr = (*r_extent_hooks)->alloc(*r_extent_hooks, new_addr,
12511256
esize, alignment, zero, commit, arena_ind_get(arena));
1257+
post_reentrancy(tsdn_tsd(tsdn));
12521258
}
12531259
if (addr == NULL) {
12541260
extent_dalloc(tsdn, arena, extent);
@@ -1486,10 +1492,13 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
14861492
err = extent_dalloc_default_impl(extent_base_get(extent),
14871493
extent_size_get(extent));
14881494
} else {
1495+
assert(!tsdn_null(tsdn));
1496+
pre_reentrancy(tsdn_tsd(tsdn));
14891497
err = ((*r_extent_hooks)->dalloc == NULL ||
14901498
(*r_extent_hooks)->dalloc(*r_extent_hooks,
14911499
extent_base_get(extent), extent_size_get(extent),
14921500
extent_committed_get(extent), arena_ind_get(arena)));
1501+
post_reentrancy(tsdn_tsd(tsdn));
14931502
}
14941503

14951504
if (!err) {
@@ -1515,6 +1524,10 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
15151524
}
15161525

15171526
extent_reregister(tsdn, extent);
1527+
if (*r_extent_hooks != &extent_hooks_default) {
1528+
assert(!tsdn_null(tsdn));
1529+
pre_reentrancy(tsdn_tsd(tsdn));
1530+
}
15181531
/* Try to decommit; purge if that fails. */
15191532
bool zeroed;
15201533
if (!extent_committed_get(extent)) {
@@ -1536,6 +1549,9 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
15361549
} else {
15371550
zeroed = false;
15381551
}
1552+
if (*r_extent_hooks != &extent_hooks_default) {
1553+
post_reentrancy(tsdn_tsd(tsdn));
1554+
}
15391555
extent_zeroed_set(extent, zeroed);
15401556

15411557
if (config_prof) {
@@ -1579,9 +1595,12 @@ extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena,
15791595
extent_destroy_default_impl(extent_base_get(extent),
15801596
extent_size_get(extent));
15811597
} else if ((*r_extent_hooks)->destroy != NULL) {
1598+
assert(!tsdn_null(tsdn));
1599+
pre_reentrancy(tsdn_tsd(tsdn));
15821600
(*r_extent_hooks)->destroy(*r_extent_hooks,
15831601
extent_base_get(extent), extent_size_get(extent),
15841602
extent_committed_get(extent), arena_ind_get(arena));
1603+
post_reentrancy(tsdn_tsd(tsdn));
15851604
}
15861605

15871606
extent_dalloc(tsdn, arena, extent);
@@ -1602,9 +1621,16 @@ extent_commit_impl(tsdn_t *tsdn, arena_t *arena,
16021621
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
16031622

16041623
extent_hooks_assure_initialized(arena, r_extent_hooks);
1624+
if (*r_extent_hooks != &extent_hooks_default) {
1625+
assert(!tsdn_null(tsdn));
1626+
pre_reentrancy(tsdn_tsd(tsdn));
1627+
}
16051628
bool err = ((*r_extent_hooks)->commit == NULL ||
16061629
(*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent),
16071630
extent_size_get(extent), offset, length, arena_ind_get(arena)));
1631+
if (*r_extent_hooks != &extent_hooks_default) {
1632+
post_reentrancy(tsdn_tsd(tsdn));
1633+
}
16081634
extent_committed_set(extent, extent_committed_get(extent) || !err);
16091635
return err;
16101636
}
@@ -1633,10 +1659,17 @@ extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
16331659

16341660
extent_hooks_assure_initialized(arena, r_extent_hooks);
16351661

1662+
if (*r_extent_hooks != &extent_hooks_default) {
1663+
assert(!tsdn_null(tsdn));
1664+
pre_reentrancy(tsdn_tsd(tsdn));
1665+
}
16361666
bool err = ((*r_extent_hooks)->decommit == NULL ||
16371667
(*r_extent_hooks)->decommit(*r_extent_hooks,
16381668
extent_base_get(extent), extent_size_get(extent), offset, length,
16391669
arena_ind_get(arena)));
1670+
if (*r_extent_hooks != &extent_hooks_default) {
1671+
post_reentrancy(tsdn_tsd(tsdn));
1672+
}
16401673
extent_committed_set(extent, extent_committed_get(extent) && err);
16411674
return err;
16421675
}
@@ -1663,10 +1696,23 @@ extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,
16631696
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
16641697

16651698
extent_hooks_assure_initialized(arena, r_extent_hooks);
1666-
return ((*r_extent_hooks)->purge_lazy == NULL ||
1667-
(*r_extent_hooks)->purge_lazy(*r_extent_hooks,
1699+
1700+
if ((*r_extent_hooks)->purge_lazy == NULL) {
1701+
return true;
1702+
}
1703+
1704+
if (*r_extent_hooks != &extent_hooks_default) {
1705+
assert(!tsdn_null(tsdn));
1706+
pre_reentrancy(tsdn_tsd(tsdn));
1707+
}
1708+
bool err = (*r_extent_hooks)->purge_lazy(*r_extent_hooks,
16681709
extent_base_get(extent), extent_size_get(extent), offset, length,
1669-
arena_ind_get(arena)));
1710+
arena_ind_get(arena));
1711+
if (*r_extent_hooks != &extent_hooks_default) {
1712+
post_reentrancy(tsdn_tsd(tsdn));
1713+
}
1714+
1715+
return err;
16701716
}
16711717

16721718
bool
@@ -1699,10 +1745,21 @@ extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena,
16991745
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
17001746

17011747
extent_hooks_assure_initialized(arena, r_extent_hooks);
1702-
return ((*r_extent_hooks)->purge_forced == NULL ||
1703-
(*r_extent_hooks)->purge_forced(*r_extent_hooks,
1748+
1749+
if ((*r_extent_hooks)->purge_forced == NULL) {
1750+
return true;
1751+
}
1752+
if (*r_extent_hooks != &extent_hooks_default) {
1753+
assert(!tsdn_null(tsdn));
1754+
pre_reentrancy(tsdn_tsd(tsdn));
1755+
}
1756+
bool err = (*r_extent_hooks)->purge_forced(*r_extent_hooks,
17041757
extent_base_get(extent), extent_size_get(extent), offset, length,
1705-
arena_ind_get(arena)));
1758+
arena_ind_get(arena));
1759+
if (*r_extent_hooks != &extent_hooks_default) {
1760+
post_reentrancy(tsdn_tsd(tsdn));
1761+
}
1762+
return err;
17061763
}
17071764

17081765
bool
@@ -1771,9 +1828,17 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena,
17711828

17721829
extent_lock2(tsdn, extent, trail);
17731830

1774-
if ((*r_extent_hooks)->split(*r_extent_hooks, extent_base_get(extent),
1831+
if (*r_extent_hooks != &extent_hooks_default) {
1832+
assert(!tsdn_null(tsdn));
1833+
pre_reentrancy(tsdn_tsd(tsdn));
1834+
}
1835+
bool err = (*r_extent_hooks)->split(*r_extent_hooks, extent_base_get(extent),
17751836
size_a + size_b, size_a, size_b, extent_committed_get(extent),
1776-
arena_ind_get(arena))) {
1837+
arena_ind_get(arena));
1838+
if (*r_extent_hooks != &extent_hooks_default) {
1839+
post_reentrancy(tsdn_tsd(tsdn));
1840+
}
1841+
if (err) {
17771842
goto label_error_c;
17781843
}
17791844

@@ -1843,10 +1908,13 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena,
18431908
err = extent_merge_default_impl(extent_base_get(a),
18441909
extent_base_get(b));
18451910
} else {
1911+
assert(!tsdn_null(tsdn));
1912+
pre_reentrancy(tsdn_tsd(tsdn));
18461913
err = (*r_extent_hooks)->merge(*r_extent_hooks,
18471914
extent_base_get(a), extent_size_get(a), extent_base_get(b),
18481915
extent_size_get(b), extent_committed_get(a),
18491916
arena_ind_get(arena));
1917+
post_reentrancy(tsdn_tsd(tsdn));
18501918
}
18511919

18521920
if (err) {

0 commit comments

Comments
 (0)