Skip to content

Commit 0ee7732

Browse files
arnaud-lbarnaud-lb
committed
Fix opcache.huge_code_pages
Building opcache into the main executable breaks opcache.huge_code_pages, as we were relying on the fact that accel_remap_huge_pages() is not in the same mapping as the main text segment. Here I ensure that accel_remap_huge_pages() is placed out of the text segment, and remap only the text segment. This approach is used in [1]. Closes phpGH-19388. [1] https://github.com/intel/iodlr/blob/676bb7dec378d561e4d900fbdaed6ce8dbe449a1/large_page-c/large_page.c#L260.
1 parent 95d52d5 commit 0ee7732

File tree

3 files changed

+115
-76
lines changed

3 files changed

+115
-76
lines changed

‎NEWS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ PHP NEWS
99
- Opcache:
1010
. Fixed bug GH-19486 (Incorrect opline after deoptimization). (Arnaud)
1111
. Fixed bug GH-19601 (Wrong JIT stack setup on aarch64/clang). (Arnaud)
12+
. Fixed bug GH-19388 (Broken opcache.huge_code_pages). (Arnaud)
1213

1314
- PCRE:
1415
. Upgraded to pcre2lib from 10.45 to 10.46. (nielsdos)

‎Zend/zend_portability.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@
8989
#ifndef __has_feature
9090
# define __has_feature(x) 0
9191
#endif
92+
#ifndef __has_include
93+
# define __has_include(x) 0
94+
#endif
9295

9396
#if defined(ZEND_WIN32) && !defined(__clang__)
9497
# define ZEND_ASSUME(c) __assume(c)
@@ -750,6 +753,10 @@ extern "C++" {
750753
# define ZEND_SET_ALIGNED(alignment, decl) decl
751754
#endif
752755

756+
#if __has_attribute(section)
757+
# define HAVE_ATTRIBUTE_SECTION
758+
#endif
759+
753760
#define ZEND_SLIDE_TO_ALIGNED(alignment, ptr) (((uintptr_t)(ptr) + ((alignment)-1)) & ~((alignment)-1))
754761
#define ZEND_SLIDE_TO_ALIGNED16(ptr) ZEND_SLIDE_TO_ALIGNED(Z_UL(16), ptr)
755762

‎ext/opcache/ZendAccelerator.c‎

Lines changed: 107 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -2974,10 +2974,22 @@ static void accel_globals_dtor(zend_accel_globals *accel_globals)
29742974
# include <sys/user.h>
29752975
# define MAP_HUGETLB MAP_ALIGNED_SUPER
29762976
# endif
2977+
# if __has_include(<link.h>)
2978+
# include <link.h>
2979+
# endif
2980+
# if __has_include(<elf.h>)
2981+
# include <elf.h>
2982+
# endif
29772983
# endif
29782984

2979-
# if defined(MAP_HUGETLB) || defined(MADV_HUGEPAGE)
2980-
static zend_result accel_remap_huge_pages(void *start, size_t size, size_t real_size, const char *name, size_t offset)
2985+
# define ZEND_HUGE_PAGE_SIZE (2UL * 1024 * 1024)
2986+
2987+
# if (defined(__linux__) || defined(__FreeBSD__)) && (defined(MAP_HUGETLB) || defined(MADV_HUGEPAGE)) && defined(HAVE_ATTRIBUTE_ALIGNED) && defined(HAVE_ATTRIBUTE_SECTION) && __has_include(<link.h>) && __has_include(<elf.h>)
2988+
static zend_result
2989+
__attribute__((section(".remap_stub")))
2990+
__attribute__((aligned(ZEND_HUGE_PAGE_SIZE)))
2991+
zend_never_inline
2992+
accel_remap_huge_pages(void *start, size_t size, size_t real_size)
29812993
{
29822994
void *ret = MAP_FAILED;
29832995
void *mem;
@@ -3030,94 +3042,113 @@ static zend_result accel_remap_huge_pages(void *start, size_t size, size_t real_
30303042

30313043
// Given the MAP_FIXED flag the address can never diverge
30323044
ZEND_ASSERT(ret == start);
3033-
zend_mmap_set_name(start, size, "zend_huge_code_pages");
3045+
30343046
memcpy(start, mem, real_size);
30353047
mprotect(start, size, PROT_READ | PROT_EXEC);
3048+
zend_mmap_set_name(start, size, "zend_huge_code_pages");
30363049

30373050
munmap(mem, size);
30383051

30393052
return SUCCESS;
30403053
}
30413054

3042-
static void accel_move_code_to_huge_pages(void)
3043-
{
3055+
static int accel_dl_iterate_phdr_callback(struct dl_phdr_info *info, size_t size, void *data) {
3056+
if (info->dlpi_name == NULL || strcmp(info->dlpi_name, "") == 0) {
3057+
*((uintptr_t*)data) = info->dlpi_addr;
3058+
return 1;
3059+
}
3060+
return 0;
3061+
}
3062+
3063+
static zend_result accel_find_program_section(ElfW(Shdr) *section) {
3064+
3065+
uintptr_t base_addr;
3066+
if (dl_iterate_phdr(accel_dl_iterate_phdr_callback, &base_addr) != 1) {
3067+
zend_error(E_WARNING, ACCELERATOR_PRODUCT_NAME ": opcache.huge_code_pages: executable base address not found");
3068+
return FAILURE;
3069+
}
3070+
30443071
#if defined(__linux__)
3045-
FILE *f;
3046-
long unsigned int huge_page_size = 2 * 1024 * 1024;
3047-
3048-
f = fopen("/proc/self/maps", "r");
3049-
if (f) {
3050-
long unsigned int start, end, offset, inode;
3051-
char perm[5], dev[10], name[MAXPATHLEN];
3052-
int ret;
3053-
extern char *__progname;
3054-
char buffer[MAXPATHLEN];
3055-
3056-
while (fgets(buffer, MAXPATHLEN, f)) {
3057-
ret = sscanf(buffer, "%lx-%lx %4s %lx %9s %lu %s\n", &start, &end, perm, &offset, dev, &inode, name);
3058-
if (ret >= 6) {
3059-
/* try to find the php text segment and map it into huge pages
3060-
Lines without 'name' are going to be skipped */
3061-
if (ret > 6 && perm[0] == 'r' && perm[1] == '-' && perm[2] == 'x' && name[0] == '/' \
3062-
&& strstr(name, __progname)) {
3063-
long unsigned int seg_start = ZEND_MM_ALIGNED_SIZE_EX(start, huge_page_size);
3064-
long unsigned int seg_end = (end & ~(huge_page_size-1L));
3065-
long unsigned int real_end;
3066-
3067-
ret = fscanf(f, "%lx-", &start);
3068-
if (ret == 1 && start == seg_end + huge_page_size) {
3069-
real_end = end;
3070-
seg_end = start;
3071-
} else {
3072-
real_end = seg_end;
3073-
}
3072+
FILE *f = fopen("/proc/self/exe", "r");
3073+
#elif defined(__FreeBSD__)
3074+
char path[4096];
3075+
int mib[4];
3076+
size_t len = sizeof(path);
30743077

3075-
if (seg_end > seg_start) {
3076-
zend_accel_error(ACCEL_LOG_DEBUG, "remap to huge page %lx-%lx %s \n", seg_start, seg_end, name);
3077-
accel_remap_huge_pages((void*)seg_start, seg_end - seg_start, real_end - seg_start, name, offset + seg_start - start);
3078-
}
3079-
break;
3080-
}
3081-
}
3082-
}
3078+
mib[0] = CTL_KERN;
3079+
mib[1] = KERN_PROC;
3080+
mib[2] = KERN_PROC_PATHNAME;
3081+
mib[3] = -1; /* Current process */
3082+
3083+
if (sysctl(mib, 4, path, &len, NULL, 0) == -1) {
3084+
zend_error(E_WARNING, ACCELERATOR_PRODUCT_NAME ": opcache.huge_code_pages: sysctl(KERN_PROC_PATHNAME) failed: %s (%d)",
3085+
strerror(errno), errno);
3086+
return FAILURE;
3087+
}
3088+
3089+
FILE *f = fopen(path, "r");
3090+
#endif
3091+
if (!f) {
3092+
zend_error(E_WARNING, ACCELERATOR_PRODUCT_NAME ": opcache.huge_code_pages: fopen(/proc/self/exe) failed: %s (%d)",
3093+
strerror(errno), errno);
3094+
return FAILURE;
3095+
}
3096+
3097+
/* Read ELF header */
3098+
ElfW(Ehdr) ehdr;
3099+
if (!fread(&ehdr, sizeof(ehdr), 1, f)) {
3100+
zend_error(E_WARNING, ACCELERATOR_PRODUCT_NAME ": opcache.huge_code_pages: fread() failed: %s (%d)",
3101+
strerror(errno), errno);
30833102
fclose(f);
3103+
return FAILURE;
30843104
}
3085-
#elif defined(__FreeBSD__)
3086-
size_t s = 0;
3087-
int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
3088-
long unsigned int huge_page_size = 2 * 1024 * 1024;
3089-
if (sysctl(mib, 4, NULL, &s, NULL, 0) == 0) {
3090-
s = s * 4 / 3;
3091-
void *addr = mmap(NULL, s, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0);
3092-
if (addr != MAP_FAILED) {
3093-
if (sysctl(mib, 4, addr, &s, NULL, 0) == 0) {
3094-
uintptr_t start = (uintptr_t)addr;
3095-
uintptr_t end = start + s;
3096-
while (start < end) {
3097-
struct kinfo_vmentry *entry = (struct kinfo_vmentry *)start;
3098-
size_t sz = entry->kve_structsize;
3099-
if (sz == 0) {
3100-
break;
3101-
}
3102-
int permflags = entry->kve_protection;
3103-
if ((permflags & KVME_PROT_READ) && !(permflags & KVME_PROT_WRITE) &&
3104-
(permflags & KVME_PROT_EXEC) && entry->kve_path[0] != '\0') {
3105-
long unsigned int seg_start = ZEND_MM_ALIGNED_SIZE_EX(start, huge_page_size);
3106-
long unsigned int seg_end = (end & ~(huge_page_size-1L));
3107-
if (seg_end > seg_start) {
3108-
zend_accel_error(ACCEL_LOG_DEBUG, "remap to huge page %lx-%lx %s \n", seg_start, seg_end, entry->kve_path);
3109-
accel_remap_huge_pages((void*)seg_start, seg_end - seg_start, seg_end - seg_start, entry->kve_path, entry->kve_offset + seg_start - start);
3110-
// First relevant segment found is our binary
3111-
break;
3112-
}
3113-
}
3114-
start += sz;
3115-
}
3116-
}
3117-
munmap(addr, s);
3105+
3106+
/* Read section headers */
3107+
ElfW(Shdr) shdrs[ehdr.e_shnum];
3108+
if (fseek(f, ehdr.e_shoff, SEEK_SET) != 0) {
3109+
zend_error(E_WARNING, ACCELERATOR_PRODUCT_NAME ": opcache.huge_code_pages: fseek() failed: %s (%d)",
3110+
strerror(errno), errno);
3111+
fclose(f);
3112+
return FAILURE;
3113+
}
3114+
if (fread(shdrs, sizeof(shdrs[0]), ehdr.e_shnum, f) != ehdr.e_shnum) {
3115+
zend_error(E_WARNING, ACCELERATOR_PRODUCT_NAME ": opcache.huge_code_pages: fread() failed: %s (%d)",
3116+
strerror(errno), errno);
3117+
fclose(f);
3118+
return FAILURE;
3119+
}
3120+
3121+
fclose(f);
3122+
3123+
/* Find the program section */
3124+
for (ElfW(Half) idx = 0; idx < ehdr.e_shnum; idx++) {
3125+
ElfW(Shdr) *sh = &shdrs[idx];
3126+
uintptr_t start = (uintptr_t)sh->sh_addr + base_addr;
3127+
zend_accel_error(ACCEL_LOG_DEBUG, "considering section %016" PRIxPTR "-%016" PRIxPTR " vs %016" PRIxPTR "\n", start, start + sh->sh_size, (uintptr_t)accel_find_program_section);
3128+
if ((uintptr_t)accel_find_program_section >= start && (uintptr_t)accel_find_program_section < start + sh->sh_size) {
3129+
*section = *sh;
3130+
section->sh_addr = (ElfW(Addr))start;
3131+
return SUCCESS;
31183132
}
31193133
}
3120-
#endif
3134+
3135+
return FAILURE;
3136+
}
3137+
3138+
static void accel_move_code_to_huge_pages(void)
3139+
{
3140+
ElfW(Shdr) section;
3141+
if (accel_find_program_section(&section) == FAILURE) {
3142+
zend_error(E_WARNING, ACCELERATOR_PRODUCT_NAME ": opcache.huge_code_pages: program section not found");
3143+
return;
3144+
}
3145+
3146+
uintptr_t start = ZEND_MM_ALIGNED_SIZE_EX(section.sh_addr, ZEND_HUGE_PAGE_SIZE);
3147+
uintptr_t end = (section.sh_addr + section.sh_size) & ~(ZEND_HUGE_PAGE_SIZE-1UL);
3148+
if (end > start) {
3149+
zend_accel_error(ACCEL_LOG_DEBUG, "remap to huge page %" PRIxPTR "-%" PRIxPTR "\n", start, end);
3150+
accel_remap_huge_pages((void*)start, end - start, end - start);
3151+
}
31213152
}
31223153
# else
31233154
static void accel_move_code_to_huge_pages(void)

0 commit comments

Comments
 (0)