Skip to content

Commit d84d290

Browse files
committed
Fix/enhance THP integration.
Detect whether chunks start off as THP-capable by default (according to the state of /sys/kernel/mm/transparent_hugepage/enabled), and use this as the basis for whether to call pages_nohuge() once per chunk during first purge of any of the chunk's page runs. Add the --disable-thp configure option, as well as the the opt.thp mallctl. This resolves jemalloc#541.
1 parent 766ddcd commit d84d290

File tree

13 files changed

+177
-21
lines changed

13 files changed

+177
-21
lines changed

INSTALL

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,13 @@ any of the following arguments (not a definitive list) to 'configure':
157157
released in bulk, thus reducing the total number of mutex operations. See
158158
the "opt.tcache" option for usage details.
159159

160+
--disable-thp
161+
Disable transparent huge page (THP) integration. On systems with THP
162+
support, THPs are explicitly disabled as a side effect of unused dirty page
163+
purging for chunks that back small and/or large allocations, because such
164+
chunks typically comprise active, unused dirty, and untouched clean
165+
pages.
166+
160167
--disable-munmap
161168
Disable virtual memory deallocation via munmap(2); instead keep track of
162169
the virtual memory for later use. munmap() is disabled by default (i.e.

configure.ac

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1683,10 +1683,31 @@ if test "x${je_cv_madvise}" = "xyes" ; then
16831683
madvise((void *)0, 0, MADV_NOHUGEPAGE);
16841684
], [je_cv_thp])
16851685
if test "x${je_cv_thp}" = "xyes" ; then
1686-
AC_DEFINE([JEMALLOC_THP], [ ])
1686+
AC_DEFINE([JEMALLOC_HAVE_MADVISE_HUGE], [ ])
16871687
fi
16881688
fi
16891689

1690+
dnl Enable transparent huge page support by default.
1691+
AC_ARG_ENABLE([thp],
1692+
[AS_HELP_STRING([--disable-thp],
1693+
[Disable transparent huge page supprot])],
1694+
[if test "x$enable_thp" = "xno" -o "x${je_cv_thp}" != "xyes" ; then
1695+
enable_thp="0"
1696+
else
1697+
enable_thp="1"
1698+
fi
1699+
],
1700+
[if test "x${je_cv_thp}" = "xyes" ; then
1701+
enable_thp="1"
1702+
else
1703+
enable_thp="0"
1704+
fi
1705+
])
1706+
if test "x$enable_thp" = "x1" ; then
1707+
AC_DEFINE([JEMALLOC_THP], [ ])
1708+
fi
1709+
AC_SUBST([enable_thp])
1710+
16901711
dnl ============================================================================
16911712
dnl Check whether __sync_{add,sub}_and_fetch() are available despite
16921713
dnl __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n macros being undefined.
@@ -2014,6 +2035,7 @@ AC_MSG_RESULT([prof-libunwind : ${enable_prof_libunwind}])
20142035
AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}])
20152036
AC_MSG_RESULT([prof-gcc : ${enable_prof_gcc}])
20162037
AC_MSG_RESULT([tcache : ${enable_tcache}])
2038+
AC_MSG_RESULT([thp : ${enable_thp}])
20172039
AC_MSG_RESULT([fill : ${enable_fill}])
20182040
AC_MSG_RESULT([utrace : ${enable_utrace}])
20192041
AC_MSG_RESULT([valgrind : ${enable_valgrind}])

doc/jemalloc.xml.in

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -850,6 +850,17 @@ for (i = 0; i < nbins; i++) {
850850
during build configuration.</para></listitem>
851851
</varlistentry>
852852

853+
<varlistentry id="config.thp">
854+
<term>
855+
<mallctl>config.thp</mallctl>
856+
(<type>bool</type>)
857+
<literal>r-</literal>
858+
</term>
859+
<listitem><para><option>--disable-thp</option> was not specified
860+
during build configuration, and the system supports transparent huge
861+
page manipulation.</para></listitem>
862+
</varlistentry>
863+
853864
<varlistentry id="config.tls">
854865
<term>
855866
<mallctl>config.tls</mallctl>
@@ -1162,6 +1173,21 @@ malloc_conf = "xmalloc:true";]]></programlisting>
11621173
forcefully disabled.</para></listitem>
11631174
</varlistentry>
11641175

1176+
<varlistentry id="opt.thp">
1177+
<term>
1178+
<mallctl>opt.thp</mallctl>
1179+
(<type>bool</type>)
1180+
<literal>r-</literal>
1181+
[<option>--enable-thp</option>]
1182+
</term>
1183+
<listitem><para>Transparent huge page (THP) integration
1184+
enabled/disabled. When enabled, THPs are explicitly disabled as a side
1185+
effect of unused dirty page purging for chunks that back small and/or
1186+
large allocations, because such chunks typically comprise active,
1187+
unused dirty, and untouched clean pages. This option is enabled by
1188+
default.</para></listitem>
1189+
</varlistentry>
1190+
11651191
<varlistentry id="opt.lg_tcache_max">
11661192
<term>
11671193
<mallctl>opt.lg_tcache_max</mallctl>

include/jemalloc/internal/arena.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -506,6 +506,7 @@ static const size_t large_pad =
506506
#endif
507507
;
508508

509+
extern bool opt_thp;
509510
extern purge_mode_t opt_purge;
510511
extern const char *purge_mode_names[];
511512
extern ssize_t opt_lg_dirty_mult;

include/jemalloc/internal/jemalloc_internal.h.in

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,13 @@ static const bool config_tcache =
9999
false
100100
#endif
101101
;
102+
static const bool config_thp =
103+
#ifdef JEMALLOC_THP
104+
true
105+
#else
106+
false
107+
#endif
108+
;
102109
static const bool config_tls =
103110
#ifdef JEMALLOC_TLS
104111
true

include/jemalloc/internal/jemalloc_internal_defs.h.in

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,12 @@
252252
/* Defined if madvise(2) is available. */
253253
#undef JEMALLOC_HAVE_MADVISE
254254

255+
/*
256+
* Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE
257+
* arguments to madvise(2).
258+
*/
259+
#undef JEMALLOC_HAVE_MADVISE_HUGE
260+
255261
/*
256262
* Methods for purging unused pages differ between operating systems.
257263
*
@@ -264,10 +270,7 @@
264270
#undef JEMALLOC_PURGE_MADVISE_FREE
265271
#undef JEMALLOC_PURGE_MADVISE_DONTNEED
266272

267-
/*
268-
* Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE
269-
* arguments to madvise(2).
270-
*/
273+
/* Defined if transparent huge page support is enabled. */
271274
#undef JEMALLOC_THP
272275

273276
/* Define if operating system has alloca.h header. */

include/jemalloc/internal/private_symbols.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,7 @@ opt_quarantine
392392
opt_redzone
393393
opt_stats_print
394394
opt_tcache
395+
opt_thp
395396
opt_utrace
396397
opt_xmalloc
397398
opt_zero

src/arena.c

Lines changed: 91 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
/******************************************************************************/
55
/* Data. */
66

7+
bool opt_thp = true;
8+
static bool thp_initially_huge;
79
purge_mode_t opt_purge = PURGE_DEFAULT;
810
const char *purge_mode_names[] = {
911
"ratio",
@@ -680,7 +682,9 @@ arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena)
680682
if (chunk == NULL)
681683
return (NULL);
682684

683-
chunk->hugepage = true;
685+
if (config_thp && opt_thp) {
686+
chunk->hugepage = thp_initially_huge;
687+
}
684688

685689
/*
686690
* Initialize the map to contain one maximal free untouched run. Mark
@@ -745,14 +749,17 @@ arena_chunk_alloc(tsdn_t *tsdn, arena_t *arena)
745749
static void
746750
arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk)
747751
{
748-
size_t sn, hugepage;
752+
size_t sn;
753+
UNUSED bool hugepage JEMALLOC_CC_SILENCE_INIT(false);
749754
bool committed;
750755
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
751756

752757
chunk_deregister(chunk, &chunk->node);
753758

754759
sn = extent_node_sn_get(&chunk->node);
755-
hugepage = chunk->hugepage;
760+
if (config_thp && opt_thp) {
761+
hugepage = chunk->hugepage;
762+
}
756763
committed = (arena_mapbits_decommitted_get(chunk, map_bias) == 0);
757764
if (!committed) {
758765
/*
@@ -765,13 +772,16 @@ arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk)
765772
chunk_hooks.decommit(chunk, chunksize, 0, map_bias << LG_PAGE,
766773
arena->ind);
767774
}
768-
if (!hugepage) {
775+
if (config_thp && opt_thp && hugepage != thp_initially_huge) {
769776
/*
770-
* Convert chunk back to the default state, so that all
771-
* subsequent chunk allocations start out with chunks that can
772-
* be backed by transparent huge pages.
777+
* Convert chunk back to initial THP state, so that all
778+
* subsequent chunk allocations start out in a consistent state.
773779
*/
774-
pages_huge(chunk, chunksize);
780+
if (thp_initially_huge) {
781+
pages_huge(chunk, chunksize);
782+
} else {
783+
pages_nohuge(chunk, chunksize);
784+
}
775785
}
776786

777787
chunk_dalloc_cache(tsdn, arena, &chunk_hooks, (void *)chunk, chunksize,
@@ -1711,13 +1721,13 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
17111721

17121722
/*
17131723
* If this is the first run purged within chunk, mark
1714-
* the chunk as non-huge. This will prevent all use of
1715-
* transparent huge pages for this chunk until the chunk
1716-
* as a whole is deallocated.
1724+
* the chunk as non-THP-capable. This will prevent all
1725+
* use of THPs for this chunk until the chunk as a whole
1726+
* is deallocated.
17171727
*/
1718-
if (chunk->hugepage) {
1719-
pages_nohuge(chunk, chunksize);
1720-
chunk->hugepage = false;
1728+
if (config_thp && opt_thp && chunk->hugepage) {
1729+
chunk->hugepage = pages_nohuge(chunk,
1730+
chunksize);
17211731
}
17221732

17231733
assert(pageind + npages <= chunk_npages);
@@ -3772,11 +3782,78 @@ bin_info_init(void)
37723782
#undef SC
37733783
}
37743784

3785+
static void
3786+
init_thp_initially_huge(void) {
3787+
int fd;
3788+
char buf[sizeof("[always] madvise never\n")];
3789+
ssize_t nread;
3790+
static const char *enabled_states[] = {
3791+
"[always] madvise never\n",
3792+
"always [madvise] never\n",
3793+
"always madvise [never]\n"
3794+
};
3795+
static const bool thp_initially_huge_states[] = {
3796+
true,
3797+
false,
3798+
false
3799+
};
3800+
unsigned i;
3801+
3802+
if (config_debug) {
3803+
for (i = 0; i < sizeof(enabled_states)/sizeof(const char *);
3804+
i++) {
3805+
assert(sizeof(buf) > strlen(enabled_states[i]));
3806+
}
3807+
}
3808+
assert(sizeof(enabled_states)/sizeof(const char *) ==
3809+
sizeof(thp_initially_huge_states)/sizeof(bool));
3810+
3811+
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
3812+
fd = (int)syscall(SYS_open,
3813+
"/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
3814+
#else
3815+
fd = open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
3816+
#endif
3817+
if (fd == -1) {
3818+
goto label_error;
3819+
}
3820+
3821+
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read)
3822+
nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf));
3823+
#else
3824+
nread = read(fd, &buf, sizeof(buf));
3825+
#endif
3826+
3827+
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
3828+
syscall(SYS_close, fd);
3829+
#else
3830+
close(fd);
3831+
#endif
3832+
3833+
if (nread < 1) {
3834+
goto label_error;
3835+
}
3836+
for (i = 0; i < sizeof(enabled_states)/sizeof(const char *);
3837+
i++) {
3838+
if (strncmp(buf, enabled_states[i], (size_t)nread) == 0) {
3839+
thp_initially_huge = thp_initially_huge_states[i];
3840+
return;
3841+
}
3842+
}
3843+
3844+
label_error:
3845+
thp_initially_huge = false;
3846+
}
3847+
37753848
void
37763849
arena_boot(void)
37773850
{
37783851
unsigned i;
37793852

3853+
if (config_thp && opt_thp) {
3854+
init_thp_initially_huge();
3855+
}
3856+
37803857
arena_lg_dirty_mult_default_set(opt_lg_dirty_mult);
37813858
arena_decay_time_default_set(opt_decay_time);
37823859

src/ctl.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ CTL_PROTO(config_prof_libgcc)
8484
CTL_PROTO(config_prof_libunwind)
8585
CTL_PROTO(config_stats)
8686
CTL_PROTO(config_tcache)
87+
CTL_PROTO(config_thp)
8788
CTL_PROTO(config_tls)
8889
CTL_PROTO(config_utrace)
8990
CTL_PROTO(config_valgrind)
@@ -104,6 +105,7 @@ CTL_PROTO(opt_utrace)
104105
CTL_PROTO(opt_xmalloc)
105106
CTL_PROTO(opt_tcache)
106107
CTL_PROTO(opt_lg_tcache_max)
108+
CTL_PROTO(opt_thp)
107109
CTL_PROTO(opt_prof)
108110
CTL_PROTO(opt_prof_prefix)
109111
CTL_PROTO(opt_prof_active)
@@ -258,6 +260,7 @@ static const ctl_named_node_t config_node[] = {
258260
{NAME("prof_libunwind"), CTL(config_prof_libunwind)},
259261
{NAME("stats"), CTL(config_stats)},
260262
{NAME("tcache"), CTL(config_tcache)},
263+
{NAME("thp"), CTL(config_thp)},
261264
{NAME("tls"), CTL(config_tls)},
262265
{NAME("utrace"), CTL(config_utrace)},
263266
{NAME("valgrind"), CTL(config_valgrind)},
@@ -281,6 +284,7 @@ static const ctl_named_node_t opt_node[] = {
281284
{NAME("xmalloc"), CTL(opt_xmalloc)},
282285
{NAME("tcache"), CTL(opt_tcache)},
283286
{NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)},
287+
{NAME("thp"), CTL(opt_thp)},
284288
{NAME("prof"), CTL(opt_prof)},
285289
{NAME("prof_prefix"), CTL(opt_prof_prefix)},
286290
{NAME("prof_active"), CTL(opt_prof_active)},
@@ -1268,6 +1272,7 @@ CTL_RO_CONFIG_GEN(config_prof_libgcc, bool)
12681272
CTL_RO_CONFIG_GEN(config_prof_libunwind, bool)
12691273
CTL_RO_CONFIG_GEN(config_stats, bool)
12701274
CTL_RO_CONFIG_GEN(config_tcache, bool)
1275+
CTL_RO_CONFIG_GEN(config_thp, bool)
12711276
CTL_RO_CONFIG_GEN(config_tls, bool)
12721277
CTL_RO_CONFIG_GEN(config_utrace, bool)
12731278
CTL_RO_CONFIG_GEN(config_valgrind, bool)
@@ -1291,6 +1296,7 @@ CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool)
12911296
CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool)
12921297
CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool)
12931298
CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t)
1299+
CTL_RO_NL_CGEN(config_thp, opt_thp, opt_thp, bool)
12941300
CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool)
12951301
CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *)
12961302
CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool)

src/jemalloc.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1272,6 +1272,9 @@ malloc_conf_init(void)
12721272
"lg_tcache_max", -1,
12731273
(sizeof(size_t) << 3) - 1)
12741274
}
1275+
if (config_thp) {
1276+
CONF_HANDLE_BOOL(opt_thp, "thp", true)
1277+
}
12751278
if (config_prof) {
12761279
CONF_HANDLE_BOOL(opt_prof, "prof", true)
12771280
CONF_HANDLE_CHAR_P(opt_prof_prefix,

0 commit comments

Comments
 (0)