44/******************************************************************************/
55/* Data. */
66
7+ bool opt_thp = true;
8+ static bool thp_initially_huge ;
79purge_mode_t opt_purge = PURGE_DEFAULT ;
810const char * purge_mode_names [] = {
911 "ratio" ,
@@ -680,7 +682,9 @@ arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena)
680682 if (chunk == NULL )
681683 return (NULL );
682684
683- chunk -> hugepage = true;
685+ if (config_thp && opt_thp ) {
686+ chunk -> hugepage = thp_initially_huge ;
687+ }
684688
685689 /*
686690 * Initialize the map to contain one maximal free untouched run. Mark
@@ -745,14 +749,17 @@ arena_chunk_alloc(tsdn_t *tsdn, arena_t *arena)
745749static void
746750arena_chunk_discard (tsdn_t * tsdn , arena_t * arena , arena_chunk_t * chunk )
747751{
748- size_t sn , hugepage ;
752+ size_t sn ;
753+ UNUSED bool hugepage JEMALLOC_CC_SILENCE_INIT (false );
749754 bool committed ;
750755 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER ;
751756
752757 chunk_deregister (chunk , & chunk -> node );
753758
754759 sn = extent_node_sn_get (& chunk -> node );
755- hugepage = chunk -> hugepage ;
760+ if (config_thp && opt_thp ) {
761+ hugepage = chunk -> hugepage ;
762+ }
756763 committed = (arena_mapbits_decommitted_get (chunk , map_bias ) == 0 );
757764 if (!committed ) {
758765 /*
@@ -765,13 +772,16 @@ arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk)
765772 chunk_hooks .decommit (chunk , chunksize , 0 , map_bias << LG_PAGE ,
766773 arena -> ind );
767774 }
768- if (! hugepage ) {
775+ if (config_thp && opt_thp && hugepage != thp_initially_huge ) {
769776 /*
770- * Convert chunk back to the default state, so that all
771- * subsequent chunk allocations start out with chunks that can
772- * be backed by transparent huge pages.
777+ * Convert chunk back to initial THP state, so that all
778+ * subsequent chunk allocations start out in a consistent state.
773779 */
774- pages_huge (chunk , chunksize );
780+ if (thp_initially_huge ) {
781+ pages_huge (chunk , chunksize );
782+ } else {
783+ pages_nohuge (chunk , chunksize );
784+ }
775785 }
776786
777787 chunk_dalloc_cache (tsdn , arena , & chunk_hooks , (void * )chunk , chunksize ,
@@ -1711,13 +1721,13 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
17111721
17121722 /*
17131723 * If this is the first run purged within chunk, mark
1714- * the chunk as non-huge . This will prevent all use of
1715- * transparent huge pages for this chunk until the chunk
1716- * as a whole is deallocated.
1724+ * the chunk as non-THP-capable . This will prevent all
1725+ * use of THPs for this chunk until the chunk as a whole
1726+ * is deallocated.
17171727 */
1718- if (chunk -> hugepage ) {
1719- pages_nohuge (chunk , chunksize );
1720- chunk -> hugepage = false ;
1728+ if (config_thp && opt_thp && chunk -> hugepage ) {
1729+ chunk -> hugepage = pages_nohuge (chunk ,
1730+ chunksize ) ;
17211731 }
17221732
17231733 assert (pageind + npages <= chunk_npages );
@@ -3772,11 +3782,78 @@ bin_info_init(void)
37723782#undef SC
37733783}
37743784
3785+ static void
3786+ init_thp_initially_huge (void ) {
3787+ int fd ;
3788+ char buf [sizeof ("[always] madvise never\n" )];
3789+ ssize_t nread ;
3790+ static const char * enabled_states [] = {
3791+ "[always] madvise never\n" ,
3792+ "always [madvise] never\n" ,
3793+ "always madvise [never]\n"
3794+ };
3795+ static const bool thp_initially_huge_states [] = {
3796+ true,
3797+ false,
3798+ false
3799+ };
3800+ unsigned i ;
3801+
3802+ if (config_debug ) {
3803+ for (i = 0 ; i < sizeof (enabled_states )/sizeof (const char * );
3804+ i ++ ) {
3805+ assert (sizeof (buf ) > strlen (enabled_states [i ]));
3806+ }
3807+ }
3808+ assert (sizeof (enabled_states )/sizeof (const char * ) ==
3809+ sizeof (thp_initially_huge_states )/sizeof (bool ));
3810+
3811+ #if defined(JEMALLOC_USE_SYSCALL ) && defined(SYS_open )
3812+ fd = (int )syscall (SYS_open ,
3813+ "/sys/kernel/mm/transparent_hugepage/enabled" , O_RDONLY );
3814+ #else
3815+ fd = open ("/sys/kernel/mm/transparent_hugepage/enabled" , O_RDONLY );
3816+ #endif
3817+ if (fd == -1 ) {
3818+ goto label_error ;
3819+ }
3820+
3821+ #if defined(JEMALLOC_USE_SYSCALL ) && defined(SYS_read )
3822+ nread = (ssize_t )syscall (SYS_read , fd , & buf , sizeof (buf ));
3823+ #else
3824+ nread = read (fd , & buf , sizeof (buf ));
3825+ #endif
3826+
3827+ #if defined(JEMALLOC_USE_SYSCALL ) && defined(SYS_close )
3828+ syscall (SYS_close , fd );
3829+ #else
3830+ close (fd );
3831+ #endif
3832+
3833+ if (nread < 1 ) {
3834+ goto label_error ;
3835+ }
3836+ for (i = 0 ; i < sizeof (enabled_states )/sizeof (const char * );
3837+ i ++ ) {
3838+ if (strncmp (buf , enabled_states [i ], (size_t )nread ) == 0 ) {
3839+ thp_initially_huge = thp_initially_huge_states [i ];
3840+ return ;
3841+ }
3842+ }
3843+
3844+ label_error :
3845+ thp_initially_huge = false;
3846+ }
3847+
37753848void
37763849arena_boot (void )
37773850{
37783851 unsigned i ;
37793852
3853+ if (config_thp && opt_thp ) {
3854+ init_thp_initially_huge ();
3855+ }
3856+
37803857 arena_lg_dirty_mult_default_set (opt_lg_dirty_mult );
37813858 arena_decay_time_default_set (opt_decay_time );
37823859
0 commit comments